xref: /llvm-project-15.0.7/openmp/runtime/src/kmp_barrier.cpp (revision 4eabb120)

/*
 * kmp_barrier.cpp
 */

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "kmp_wait_release.h"
#include "kmp_barrier.h"
#include "kmp_itt.h"
#include "kmp_os.h"
#include "kmp_stats.h"
#include "ompt-specific.h"
// for distributed barrier
#include "kmp_affinity.h"

#if KMP_MIC
#include <immintrin.h>
#define USE_NGO_STORES 1
#endif // KMP_MIC

#include "tsan_annotations.h"

#if KMP_MIC && USE_NGO_STORES
// ICV copying
#define ngo_load(src) __m512d Vt = _mm512_load_pd((void *)(src))
#define ngo_store_icvs(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt)
#define ngo_store_go(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt)
#define ngo_sync() __asm__ volatile("lock; addl $0,0(%%rsp)" ::: "memory")
#else
#define ngo_load(src) ((void)0)
#define ngo_store_icvs(dst, src) copy_icvs((dst), (src))
#define ngo_store_go(dst, src) KMP_MEMCPY((dst), (src), CACHE_LINE)
#define ngo_sync() ((void)0)
#endif /* KMP_MIC && USE_NGO_STORES */

void __kmp_print_structure(void); // Forward declaration

// ---------------------------- Barrier Algorithms ----------------------------
// Distributed barrier

// Compute how many threads to have polling each cache-line.
// We want to limit the number of writes to IDEAL_GO_RESOLUTION.
void distributedBarrier::computeVarsForN(size_t n) {
  int nsockets = 1;
  if (__kmp_topology) {
    int socket_level = __kmp_topology->get_level(KMP_HW_SOCKET);
    int core_level = __kmp_topology->get_level(KMP_HW_CORE);
    int ncores_per_socket =
        __kmp_topology->calculate_ratio(core_level, socket_level);
    nsockets = __kmp_topology->get_count(socket_level);

    if (nsockets <= 0)
      nsockets = 1;
    if (ncores_per_socket <= 0)
      ncores_per_socket = 1;

    threads_per_go = ncores_per_socket >> 1;
    if (!fix_threads_per_go) {
      // Minimize num_gos
      if (threads_per_go > 4) {
        if (KMP_OPTIMIZE_FOR_REDUCTIONS) {
          threads_per_go = threads_per_go >> 1;
        }
        if (threads_per_go > 4 && nsockets == 1)
          threads_per_go = threads_per_go >> 1;
      }
    }
    if (threads_per_go == 0)
      threads_per_go = 1;
    fix_threads_per_go = true;
    num_gos = n / threads_per_go;
    if (n % threads_per_go)
      num_gos++;
    if (nsockets == 1 || num_gos == 1)
      num_groups = 1;
    else {
      num_groups = num_gos / nsockets;
      if (num_gos % nsockets)
        num_groups++;
    }
    if (num_groups <= 0)
      num_groups = 1;
    gos_per_group = num_gos / num_groups;
    if (num_gos % num_groups)
      gos_per_group++;
    threads_per_group = threads_per_go * gos_per_group;
  } else {
    num_gos = n / threads_per_go;
    if (n % threads_per_go)
      num_gos++;
    if (num_gos == 1)
      num_groups = 1;
    else {
      num_groups = num_gos / 2;
      if (num_gos % 2)
        num_groups++;
    }
    gos_per_group = num_gos / num_groups;
    if (num_gos % num_groups)
      gos_per_group++;
    threads_per_group = threads_per_go * gos_per_group;
  }
}

void distributedBarrier::computeGo(size_t n) {
  // Minimize num_gos
  for (num_gos = 1;; num_gos++)
    if (IDEAL_CONTENTION * num_gos >= n)
      break;
  threads_per_go = n / num_gos;
  if (n % num_gos)
    threads_per_go++;
  while (num_gos > MAX_GOS) {
    threads_per_go++;
    num_gos = n / threads_per_go;
    if (n % threads_per_go)
      num_gos++;
  }
  computeVarsForN(n);
}

// This function is to resize the barrier arrays when the new number of threads
// exceeds max_threads, which is the current size of all the arrays
void distributedBarrier::resize(size_t nthr) {
  KMP_DEBUG_ASSERT(nthr > max_threads);

  // expand to requested size * 2
  max_threads = nthr * 2;

  // allocate arrays to new max threads
  for (int i = 0; i < MAX_ITERS; ++i) {
    if (flags[i])
      flags[i] = (flags_s *)KMP_INTERNAL_REALLOC(flags[i],
                                                 max_threads * sizeof(flags_s));
    else
      flags[i] = (flags_s *)KMP_INTERNAL_MALLOC(max_threads * sizeof(flags_s));
  }

  if (go)
    go = (go_s *)KMP_INTERNAL_REALLOC(go, max_threads * sizeof(go_s));
  else
    go = (go_s *)KMP_INTERNAL_MALLOC(max_threads * sizeof(go_s));

  if (iter)
    iter = (iter_s *)KMP_INTERNAL_REALLOC(iter, max_threads * sizeof(iter_s));
  else
    iter = (iter_s *)KMP_INTERNAL_MALLOC(max_threads * sizeof(iter_s));

  if (sleep)
    sleep =
        (sleep_s *)KMP_INTERNAL_REALLOC(sleep, max_threads * sizeof(sleep_s));
  else
    sleep = (sleep_s *)KMP_INTERNAL_MALLOC(max_threads * sizeof(sleep_s));
}

// This function is to set all the go flags that threads might be waiting
// on, and when blocktime is not infinite, it should be followed by a wake-up
// call to each thread
kmp_uint64 distributedBarrier::go_release() {
  kmp_uint64 next_go = iter[0].iter + distributedBarrier::MAX_ITERS;
  for (size_t j = 0; j < num_gos; j++) {
    go[j].go.store(next_go);
  }
  return next_go;
}

void distributedBarrier::go_reset() {
  for (size_t j = 0; j < max_threads; ++j) {
    for (size_t i = 0; i < distributedBarrier::MAX_ITERS; ++i) {
      flags[i][j].stillNeed = 1;
    }
    go[j].go.store(0);
    iter[j].iter = 0;
  }
}

// This function inits/re-inits the distributed barrier for a particular number
// of threads. If a resize of arrays is needed, it calls the resize function.
void distributedBarrier::init(size_t nthr) {
  size_t old_max = max_threads;
  if (nthr > max_threads) { // need more space in arrays
    resize(nthr);
  }

  for (size_t i = 0; i < max_threads; i++) {
    for (size_t j = 0; j < distributedBarrier::MAX_ITERS; j++) {
      flags[j][i].stillNeed = 1;
    }
    go[i].go.store(0);
    iter[i].iter = 0;
    if (i >= old_max)
      sleep[i].sleep = false;
  }

  // Recalculate num_gos, etc. based on new nthr
  computeVarsForN(nthr);

  num_threads = nthr;

  if (team_icvs == NULL)
    team_icvs = __kmp_allocate(sizeof(kmp_internal_control_t));
}

// This function is used only when KMP_BLOCKTIME is not infinite.
// static
void __kmp_dist_barrier_wakeup(enum barrier_type bt, kmp_team_t *team,
                               size_t start, size_t stop, size_t inc,
                               size_t tid) {
  KMP_DEBUG_ASSERT(__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME);
  if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
    return;

  kmp_info_t **other_threads = team->t.t_threads;
  for (size_t thr = start; thr < stop; thr += inc) {
    KMP_DEBUG_ASSERT(other_threads[thr]);
    int gtid = other_threads[thr]->th.th_info.ds.ds_gtid;
    // Wake up worker regardless of if it appears to be sleeping or not
    __kmp_atomic_resume_64(gtid, (kmp_atomic_flag_64<> *)NULL);
  }
}

static void __kmp_dist_barrier_gather(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_dist_gather);
  kmp_team_t *team;
  distributedBarrier *b;
  kmp_info_t **other_threads;
  kmp_uint64 my_current_iter, my_next_iter;
  kmp_uint32 nproc;
  bool group_leader;

  team = this_thr->th.th_team;
  nproc = this_thr->th.th_team_nproc;
  other_threads = team->t.t_threads;
  b = team->t.b;
  my_current_iter = b->iter[tid].iter;
  my_next_iter = (my_current_iter + 1) % distributedBarrier::MAX_ITERS;
  group_leader = ((tid % b->threads_per_group) == 0);

  KA_TRACE(20,
           ("__kmp_dist_barrier_gather: T#%d(%d:%d) enter; barrier type %d\n",
            gtid, team->t.t_id, tid, bt));

#if USE_ITT_BUILD && USE_ITT_NOTIFY
  // Barrier imbalance - save arrive time to the thread
  if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
    this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time =
        __itt_get_timestamp();
  }
#endif

  if (group_leader) {
    // Start from the thread after the group leader
    size_t group_start = tid + 1;
    size_t group_end = tid + b->threads_per_group;
    size_t threads_pending = 0;

    if (group_end > nproc)
      group_end = nproc;
    do { // wait for threads in my group
      threads_pending = 0;
      // Check all the flags every time to avoid branch misspredict
      for (size_t thr = group_start; thr < group_end; thr++) {
        // Each thread uses a different cache line
        threads_pending += b->flags[my_current_iter][thr].stillNeed;
      }
      // Execute tasks here
      if (__kmp_tasking_mode != tskm_immediate_exec) {
        kmp_task_team_t *task_team = this_thr->th.th_task_team;
        if (task_team != NULL) {
          if (TCR_SYNC_4(task_team->tt.tt_active)) {
            if (KMP_TASKING_ENABLED(task_team)) {
              int tasks_completed = FALSE;
              __kmp_atomic_execute_tasks_64(
                  this_thr, gtid, (kmp_atomic_flag_64<> *)NULL, FALSE,
                  &tasks_completed USE_ITT_BUILD_ARG(itt_sync_obj), 0);
            } else
              this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
          }
        } else {
          this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
        } // if
      }
      if (TCR_4(__kmp_global.g.g_done)) {
        if (__kmp_global.g.g_abort)
          __kmp_abort_thread();
        break;
      } else if (__kmp_tasking_mode != tskm_immediate_exec &&
                 this_thr->th.th_reap_state == KMP_SAFE_TO_REAP) {
        this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
      }
    } while (threads_pending > 0);

    if (reduce) { // Perform reduction if needed
      OMPT_REDUCTION_DECL(this_thr, gtid);
      OMPT_REDUCTION_BEGIN;
      // Group leader reduces all threads in group
      for (size_t thr = group_start; thr < group_end; thr++) {
        (*reduce)(this_thr->th.th_local.reduce_data,
                  other_threads[thr]->th.th_local.reduce_data);
      }
      OMPT_REDUCTION_END;
    }

    // Set flag for next iteration
    b->flags[my_next_iter][tid].stillNeed = 1;
    // Each thread uses a different cache line; resets stillNeed to 0 to
    // indicate it has reached the barrier
    b->flags[my_current_iter][tid].stillNeed = 0;

    do { // wait for all group leaders
      threads_pending = 0;
      for (size_t thr = 0; thr < nproc; thr += b->threads_per_group) {
        threads_pending += b->flags[my_current_iter][thr].stillNeed;
      }
      // Execute tasks here
      if (__kmp_tasking_mode != tskm_immediate_exec) {
        kmp_task_team_t *task_team = this_thr->th.th_task_team;
        if (task_team != NULL) {
          if (TCR_SYNC_4(task_team->tt.tt_active)) {
            if (KMP_TASKING_ENABLED(task_team)) {
              int tasks_completed = FALSE;
              __kmp_atomic_execute_tasks_64(
                  this_thr, gtid, (kmp_atomic_flag_64<> *)NULL, FALSE,
                  &tasks_completed USE_ITT_BUILD_ARG(itt_sync_obj), 0);
            } else
              this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
          }
        } else {
          this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
        } // if
      }
      if (TCR_4(__kmp_global.g.g_done)) {
        if (__kmp_global.g.g_abort)
          __kmp_abort_thread();
        break;
      } else if (__kmp_tasking_mode != tskm_immediate_exec &&
                 this_thr->th.th_reap_state == KMP_SAFE_TO_REAP) {
        this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
      }
    } while (threads_pending > 0);

    if (reduce) { // Perform reduction if needed
      if (KMP_MASTER_TID(tid)) { // Master reduces over group leaders
        OMPT_REDUCTION_DECL(this_thr, gtid);
        OMPT_REDUCTION_BEGIN;
        for (size_t thr = b->threads_per_group; thr < nproc;
             thr += b->threads_per_group) {
          (*reduce)(this_thr->th.th_local.reduce_data,
                    other_threads[thr]->th.th_local.reduce_data);
        }
        OMPT_REDUCTION_END;
      }
    }
  } else {
    // Set flag for next iteration
    b->flags[my_next_iter][tid].stillNeed = 1;
    // Each thread uses a different cache line; resets stillNeed to 0 to
    // indicate it has reached the barrier
    b->flags[my_current_iter][tid].stillNeed = 0;
  }

  KMP_MFENCE();

  KA_TRACE(20,
           ("__kmp_dist_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
            gtid, team->t.t_id, tid, bt));
}

static void __kmp_dist_barrier_release(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_dist_release);
  kmp_team_t *team;
  distributedBarrier *b;
  kmp_bstate_t *thr_bar;
  kmp_uint64 my_current_iter, next_go;
  size_t my_go_index;
  bool group_leader;

  KA_TRACE(20, ("__kmp_dist_barrier_release: T#%d(%d) enter; barrier type %d\n",
                gtid, tid, bt));

  thr_bar = &this_thr->th.th_bar[bt].bb;

  if (!KMP_MASTER_TID(tid)) {
    // workers and non-master group leaders need to check their presence in team
    do {
      if (this_thr->th.th_used_in_team.load() != 1 &&
          this_thr->th.th_used_in_team.load() != 3) {
        // Thread is not in use in a team. Wait on location in tid's thread
        // struct. The 0 value tells anyone looking that this thread is spinning
        // or sleeping until this location becomes 3 again; 3 is the transition
        // state to get to 1 which is waiting on go and being in the team
        kmp_flag_32<false, false> my_flag(&(this_thr->th.th_used_in_team), 3);
        if (KMP_COMPARE_AND_STORE_ACQ32(&(this_thr->th.th_used_in_team), 2,
                                        0) ||
            this_thr->th.th_used_in_team.load() == 0) {
          my_flag.wait(this_thr, true USE_ITT_BUILD_ARG(itt_sync_obj));
        }
#if USE_ITT_BUILD && USE_ITT_NOTIFY
        if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
          // In fork barrier where we could not get the object reliably
          itt_sync_obj =
              __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
          // Cancel wait on previous parallel region...
          __kmp_itt_task_starting(itt_sync_obj);

          if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
            return;

          itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
          if (itt_sync_obj != NULL)
            // Call prepare as early as possible for "new" barrier
            __kmp_itt_task_finished(itt_sync_obj);
        } else
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
            if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
          return;
      }
      if (this_thr->th.th_used_in_team.load() != 1 &&
          this_thr->th.th_used_in_team.load() != 3) // spurious wake-up?
        continue;
      if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
        return;

      // At this point, the thread thinks it is in use in a team, or in
      // transition to be used in a team, but it might have reached this barrier
      // before it was marked unused by the team. Unused threads are awoken and
      // shifted to wait on local thread struct elsewhere. It also might reach
      // this point by being picked up for use by a different team. Either way,
      // we need to update the tid.
      tid = __kmp_tid_from_gtid(gtid);
      team = this_thr->th.th_team;
      KMP_DEBUG_ASSERT(tid >= 0);
      KMP_DEBUG_ASSERT(team);
      b = team->t.b;
      my_current_iter = b->iter[tid].iter;
      next_go = my_current_iter + distributedBarrier::MAX_ITERS;
      my_go_index = tid / b->threads_per_go;
      if (this_thr->th.th_used_in_team.load() == 3) {
        KMP_COMPARE_AND_STORE_ACQ32(&(this_thr->th.th_used_in_team), 3, 1);
      }
      // Check if go flag is set
      if (b->go[my_go_index].go.load() != next_go) {
        // Wait on go flag on team
        kmp_atomic_flag_64<false, true> my_flag(
            &(b->go[my_go_index].go), next_go, &(b->sleep[tid].sleep));
        my_flag.wait(this_thr, true USE_ITT_BUILD_ARG(itt_sync_obj));
        KMP_DEBUG_ASSERT(my_current_iter == b->iter[tid].iter ||
                         b->iter[tid].iter == 0);
        KMP_DEBUG_ASSERT(b->sleep[tid].sleep == false);
      }

      if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
        return;
      // At this point, the thread's go location was set. This means the primary
      // thread is safely in the barrier, and so this thread's data is
      // up-to-date, but we should check again that this thread is really in
      // use in the team, as it could have been woken up for the purpose of
      // changing team size, or reaping threads at shutdown.
      if (this_thr->th.th_used_in_team.load() == 1)
        break;
    } while (1);

    if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
      return;

    group_leader = ((tid % b->threads_per_group) == 0);
    if (group_leader) {
      // Tell all the threads in my group they can go!
      for (size_t go_idx = my_go_index + 1;
           go_idx < my_go_index + b->gos_per_group; go_idx++) {
        b->go[go_idx].go.store(next_go);
      }
      // Fence added so that workers can see changes to go. sfence inadequate.
      KMP_MFENCE();
    }

#if KMP_BARRIER_ICV_PUSH
    if (propagate_icvs) { // copy ICVs to final dest
      __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team,
                               tid, FALSE);
      copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
                (kmp_internal_control_t *)team->t.b->team_icvs);
      copy_icvs(&thr_bar->th_fixed_icvs,
                &team->t.t_implicit_task_taskdata[tid].td_icvs);
    }
#endif
    if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME && group_leader) {
      // This thread is now awake and participating in the barrier;
      // wake up the other threads in the group
      size_t nproc = this_thr->th.th_team_nproc;
      size_t group_end = tid + b->threads_per_group;
      if (nproc < group_end)
        group_end = nproc;
      __kmp_dist_barrier_wakeup(bt, team, tid + 1, group_end, 1, tid);
    }
  } else { //  Primary thread
    team = this_thr->th.th_team;
    b = team->t.b;
    my_current_iter = b->iter[tid].iter;
    next_go = my_current_iter + distributedBarrier::MAX_ITERS;
#if KMP_BARRIER_ICV_PUSH
    if (propagate_icvs) {
      // primary thread has ICVs in final destination; copy
      copy_icvs(&thr_bar->th_fixed_icvs,
                &team->t.t_implicit_task_taskdata[tid].td_icvs);
    }
#endif
    // Tell all the group leaders they can go!
    for (size_t go_idx = 0; go_idx < b->num_gos; go_idx += b->gos_per_group) {
      b->go[go_idx].go.store(next_go);
    }

    if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
      // Wake-up the group leaders
      size_t nproc = this_thr->th.th_team_nproc;
      __kmp_dist_barrier_wakeup(bt, team, tid + b->threads_per_group, nproc,
                                b->threads_per_group, tid);
    }

    // Tell all the threads in my group they can go!
    for (size_t go_idx = 1; go_idx < b->gos_per_group; go_idx++) {
      b->go[go_idx].go.store(next_go);
    }

    // Fence added so that workers can see changes to go. sfence inadequate.
    KMP_MFENCE();

    if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
      // Wake-up the other threads in my group
      size_t nproc = this_thr->th.th_team_nproc;
      size_t group_end = tid + b->threads_per_group;
      if (nproc < group_end)
        group_end = nproc;
      __kmp_dist_barrier_wakeup(bt, team, tid + 1, group_end, 1, tid);
    }
  }
  // Update to next iteration
  KMP_ASSERT(my_current_iter == b->iter[tid].iter);
  b->iter[tid].iter = (b->iter[tid].iter + 1) % distributedBarrier::MAX_ITERS;

  KA_TRACE(
      20, ("__kmp_dist_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
           gtid, team->t.t_id, tid, bt));
}

// Linear Barrier
template <bool cancellable = false>
static bool __kmp_linear_barrier_gather_template(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_gather);
  kmp_team_t *team = this_thr->th.th_team;
  kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
  kmp_info_t **other_threads = team->t.t_threads;

  KA_TRACE(
      20,
      ("__kmp_linear_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
       gtid, team->t.t_id, tid, bt));
  KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);

#if USE_ITT_BUILD && USE_ITT_NOTIFY
  // Barrier imbalance - save arrive time to the thread
  if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
    this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time =
        __itt_get_timestamp();
  }
#endif
  // We now perform a linear reduction to signal that all of the threads have
  // arrived.
  if (!KMP_MASTER_TID(tid)) {
    KA_TRACE(20,
             ("__kmp_linear_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d)"
              "arrived(%p): %llu => %llu\n",
              gtid, team->t.t_id, tid, __kmp_gtid_from_tid(0, team),
              team->t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived,
              thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
    // Mark arrival to primary thread
    /* After performing this write, a worker thread may not assume that the team
       is valid any more - it could be deallocated by the primary thread at any
       time. */
    ANNOTATE_BARRIER_BEGIN(this_thr);
    kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[0]);
    flag.release();
  } else {
    kmp_balign_team_t *team_bar = &team->t.t_bar[bt];
    int nproc = this_thr->th.th_team_nproc;
    int i;
    // Don't have to worry about sleep bit here or atomic since team setting
    kmp_uint64 new_state = team_bar->b_arrived + KMP_BARRIER_STATE_BUMP;

    // Collect all the worker team member threads.
    for (i = 1; i < nproc; ++i) {
#if KMP_CACHE_MANAGE
      // Prefetch next thread's arrived count
      if (i + 1 < nproc)
        KMP_CACHE_PREFETCH(&other_threads[i + 1]->th.th_bar[bt].bb.b_arrived);
#endif /* KMP_CACHE_MANAGE */
      KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%d) "
                    "arrived(%p) == %llu\n",
                    gtid, team->t.t_id, tid, __kmp_gtid_from_tid(i, team),
                    team->t.t_id, i,
                    &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state));

      // Wait for worker thread to arrive
      if (cancellable) {
        kmp_flag_64<true, false> flag(
            &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state);
        if (flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)))
          return true;
      } else {
        kmp_flag_64<> flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived,
                           new_state);
        flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
      }
      ANNOTATE_BARRIER_END(other_threads[i]);
#if USE_ITT_BUILD && USE_ITT_NOTIFY
      // Barrier imbalance - write min of the thread time and the other thread
      // time to the thread.
      if (__kmp_forkjoin_frames_mode == 2) {
        this_thr->th.th_bar_min_time = KMP_MIN(
            this_thr->th.th_bar_min_time, other_threads[i]->th.th_bar_min_time);
      }
#endif
      if (reduce) {
        KA_TRACE(100,
                 ("__kmp_linear_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
                  gtid, team->t.t_id, tid, __kmp_gtid_from_tid(i, team),
                  team->t.t_id, i));
        ANNOTATE_REDUCE_AFTER(reduce);
        OMPT_REDUCTION_DECL(this_thr, gtid);
        OMPT_REDUCTION_BEGIN;
        (*reduce)(this_thr->th.th_local.reduce_data,
                  other_threads[i]->th.th_local.reduce_data);
        OMPT_REDUCTION_END;
        ANNOTATE_REDUCE_BEFORE(reduce);
        ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
      }
    }
    // Don't have to worry about sleep bit here or atomic since team setting
    team_bar->b_arrived = new_state;
    KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) set team %d "
                  "arrived(%p) = %llu\n",
                  gtid, team->t.t_id, tid, team->t.t_id, &team_bar->b_arrived,
                  new_state));
  }
  KA_TRACE(
      20,
      ("__kmp_linear_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
       gtid, team->t.t_id, tid, bt));
  return false;
}

template <bool cancellable = false>
static bool __kmp_linear_barrier_release_template(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_release);
  kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
  kmp_team_t *team;

  if (KMP_MASTER_TID(tid)) {
    unsigned int i;
    kmp_uint32 nproc = this_thr->th.th_team_nproc;
    kmp_info_t **other_threads;

    team = __kmp_threads[gtid]->th.th_team;
    KMP_DEBUG_ASSERT(team != NULL);
    other_threads = team->t.t_threads;

    KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d(%d:%d) primary enter for "
                  "barrier type %d\n",
                  gtid, team->t.t_id, tid, bt));

    if (nproc > 1) {
#if KMP_BARRIER_ICV_PUSH
      {
        KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
        if (propagate_icvs) {
          ngo_load(&team->t.t_implicit_task_taskdata[0].td_icvs);
          for (i = 1; i < nproc; ++i) {
            __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[i],
                                     team, i, FALSE);
            ngo_store_icvs(&team->t.t_implicit_task_taskdata[i].td_icvs,
                           &team->t.t_implicit_task_taskdata[0].td_icvs);
          }
          ngo_sync();
        }
      }
#endif // KMP_BARRIER_ICV_PUSH

      // Now, release all of the worker threads
      for (i = 1; i < nproc; ++i) {
#if KMP_CACHE_MANAGE
        // Prefetch next thread's go flag
        if (i + 1 < nproc)
          KMP_CACHE_PREFETCH(&other_threads[i + 1]->th.th_bar[bt].bb.b_go);
#endif /* KMP_CACHE_MANAGE */
        KA_TRACE(
            20,
            ("__kmp_linear_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%d) "
             "go(%p): %u => %u\n",
             gtid, team->t.t_id, tid, other_threads[i]->th.th_info.ds.ds_gtid,
             team->t.t_id, i, &other_threads[i]->th.th_bar[bt].bb.b_go,
             other_threads[i]->th.th_bar[bt].bb.b_go,
             other_threads[i]->th.th_bar[bt].bb.b_go + KMP_BARRIER_STATE_BUMP));
        ANNOTATE_BARRIER_BEGIN(other_threads[i]);
        kmp_flag_64<> flag(&other_threads[i]->th.th_bar[bt].bb.b_go,
                           other_threads[i]);
        flag.release();
      }
    }
  } else { // Wait for the PRIMARY thread to release us
    KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d wait go(%p) == %u\n",
                  gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
    if (cancellable) {
      kmp_flag_64<true, false> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
      if (flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)))
        return true;
    } else {
      kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
      flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
    }
    ANNOTATE_BARRIER_END(this_thr);
#if USE_ITT_BUILD && USE_ITT_NOTIFY
    if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
      // In a fork barrier; cannot get the object reliably (or ITTNOTIFY is
      // disabled)
      itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
      // Cancel wait on previous parallel region...
      __kmp_itt_task_starting(itt_sync_obj);

      if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
        return false;

      itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
      if (itt_sync_obj != NULL)
        // Call prepare as early as possible for "new" barrier
        __kmp_itt_task_finished(itt_sync_obj);
    } else
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
        // Early exit for reaping threads releasing forkjoin barrier
        if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
      return false;
// The worker thread may now assume that the team is valid.
#ifdef KMP_DEBUG
    tid = __kmp_tid_from_gtid(gtid);
    team = __kmp_threads[gtid]->th.th_team;
#endif
    KMP_DEBUG_ASSERT(team != NULL);
    TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
    KA_TRACE(20,
             ("__kmp_linear_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
              gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
    KMP_MB(); // Flush all pending memory write invalidates.
  }
  KA_TRACE(
      20,
      ("__kmp_linear_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
       gtid, team->t.t_id, tid, bt));
  return false;
}

static void __kmp_linear_barrier_gather(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  __kmp_linear_barrier_gather_template<false>(
      bt, this_thr, gtid, tid, reduce USE_ITT_BUILD_ARG(itt_sync_obj));
}

static bool __kmp_linear_barrier_gather_cancellable(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  return __kmp_linear_barrier_gather_template<true>(
      bt, this_thr, gtid, tid, reduce USE_ITT_BUILD_ARG(itt_sync_obj));
}

static void __kmp_linear_barrier_release(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  __kmp_linear_barrier_release_template<false>(
      bt, this_thr, gtid, tid, propagate_icvs USE_ITT_BUILD_ARG(itt_sync_obj));
}

static bool __kmp_linear_barrier_release_cancellable(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  return __kmp_linear_barrier_release_template<true>(
      bt, this_thr, gtid, tid, propagate_icvs USE_ITT_BUILD_ARG(itt_sync_obj));
}

// Tree barrier
static void __kmp_tree_barrier_gather(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_tree_gather);
  kmp_team_t *team = this_thr->th.th_team;
  kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
  kmp_info_t **other_threads = team->t.t_threads;
  kmp_uint32 nproc = this_thr->th.th_team_nproc;
  kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt];
  kmp_uint32 branch_factor = 1 << branch_bits;
  kmp_uint32 child;
  kmp_uint32 child_tid;
  kmp_uint64 new_state = 0;

  KA_TRACE(
      20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
           gtid, team->t.t_id, tid, bt));
  KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);

#if USE_ITT_BUILD && USE_ITT_NOTIFY
  // Barrier imbalance - save arrive time to the thread
  if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
    this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time =
        __itt_get_timestamp();
  }
#endif
  // Perform tree gather to wait until all threads have arrived; reduce any
  // required data as we go
  child_tid = (tid << branch_bits) + 1;
  if (child_tid < nproc) {
    // Parent threads wait for all their children to arrive
    new_state = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
    child = 1;
    do {
      kmp_info_t *child_thr = other_threads[child_tid];
      kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
#if KMP_CACHE_MANAGE
      // Prefetch next thread's arrived count
      if (child + 1 <= branch_factor && child_tid + 1 < nproc)
        KMP_CACHE_PREFETCH(
            &other_threads[child_tid + 1]->th.th_bar[bt].bb.b_arrived);
#endif /* KMP_CACHE_MANAGE */
      KA_TRACE(20,
               ("__kmp_tree_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) "
                "arrived(%p) == %llu\n",
                gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
      // Wait for child to arrive
      kmp_flag_64<> flag(&child_bar->b_arrived, new_state);
      flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
      ANNOTATE_BARRIER_END(child_thr);
#if USE_ITT_BUILD && USE_ITT_NOTIFY
      // Barrier imbalance - write min of the thread time and a child time to
      // the thread.
      if (__kmp_forkjoin_frames_mode == 2) {
        this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
                                               child_thr->th.th_bar_min_time);
      }
#endif
      if (reduce) {
        KA_TRACE(100,
                 ("__kmp_tree_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                  gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                  team->t.t_id, child_tid));
        ANNOTATE_REDUCE_AFTER(reduce);
        OMPT_REDUCTION_DECL(this_thr, gtid);
        OMPT_REDUCTION_BEGIN;
        (*reduce)(this_thr->th.th_local.reduce_data,
                  child_thr->th.th_local.reduce_data);
        OMPT_REDUCTION_END;
        ANNOTATE_REDUCE_BEFORE(reduce);
        ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
      }
      child++;
      child_tid++;
    } while (child <= branch_factor && child_tid < nproc);
  }

  if (!KMP_MASTER_TID(tid)) { // Worker threads
    kmp_int32 parent_tid = (tid - 1) >> branch_bits;

    KA_TRACE(20,
             ("__kmp_tree_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) "
              "arrived(%p): %llu => %llu\n",
              gtid, team->t.t_id, tid, __kmp_gtid_from_tid(parent_tid, team),
              team->t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
              thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));

    // Mark arrival to parent thread
    /* After performing this write, a worker thread may not assume that the team
       is valid any more - it could be deallocated by the primary thread at any
       time.  */
    ANNOTATE_BARRIER_BEGIN(this_thr);
    kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[parent_tid]);
    flag.release();
  } else {
    // Need to update the team arrived pointer if we are the primary thread
    if (nproc > 1) // New value was already computed above
      team->t.t_bar[bt].b_arrived = new_state;
    else
      team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
    KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) set team %d "
                  "arrived(%p) = %llu\n",
                  gtid, team->t.t_id, tid, team->t.t_id,
                  &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
  }
  KA_TRACE(20,
           ("__kmp_tree_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
            gtid, team->t.t_id, tid, bt));
}

static void __kmp_tree_barrier_release(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_tree_release);
  kmp_team_t *team;
  kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
  kmp_uint32 nproc;
  kmp_uint32 branch_bits = __kmp_barrier_release_branch_bits[bt];
  kmp_uint32 branch_factor = 1 << branch_bits;
  kmp_uint32 child;
  kmp_uint32 child_tid;

  // Perform a tree release for all of the threads that have been gathered
  if (!KMP_MASTER_TID(
          tid)) { // Handle fork barrier workers who aren't part of a team yet
    KA_TRACE(20, ("__kmp_tree_barrier_release: T#%d wait go(%p) == %u\n", gtid,
                  &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
    // Wait for parent thread to release us
    kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
    flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
    ANNOTATE_BARRIER_END(this_thr);
#if USE_ITT_BUILD && USE_ITT_NOTIFY
    if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
      // In fork barrier where we could not get the object reliably (or
      // ITTNOTIFY is disabled)
      itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
      // Cancel wait on previous parallel region...
      __kmp_itt_task_starting(itt_sync_obj);

      if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
        return;

      itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
      if (itt_sync_obj != NULL)
        // Call prepare as early as possible for "new" barrier
        __kmp_itt_task_finished(itt_sync_obj);
    } else
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
        // Early exit for reaping threads releasing forkjoin barrier
        if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
      return;

    // The worker thread may now assume that the team is valid.
    team = __kmp_threads[gtid]->th.th_team;
    KMP_DEBUG_ASSERT(team != NULL);
    tid = __kmp_tid_from_gtid(gtid);

    TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
    KA_TRACE(20,
             ("__kmp_tree_barrier_release: T#%d(%d:%d) set go(%p) = %u\n", gtid,
              team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
    KMP_MB(); // Flush all pending memory write invalidates.
  } else {
    team = __kmp_threads[gtid]->th.th_team;
    KMP_DEBUG_ASSERT(team != NULL);
    KA_TRACE(20, ("__kmp_tree_barrier_release: T#%d(%d:%d) primary enter for "
                  "barrier type %d\n",
                  gtid, team->t.t_id, tid, bt));
  }
  nproc = this_thr->th.th_team_nproc;
  child_tid = (tid << branch_bits) + 1;

  if (child_tid < nproc) {
    kmp_info_t **other_threads = team->t.t_threads;
    child = 1;
    // Parent threads release all their children
    do {
      kmp_info_t *child_thr = other_threads[child_tid];
      kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
#if KMP_CACHE_MANAGE
      // Prefetch next thread's go count
      if (child + 1 <= branch_factor && child_tid + 1 < nproc)
        KMP_CACHE_PREFETCH(
            &other_threads[child_tid + 1]->th.th_bar[bt].bb.b_go);
#endif /* KMP_CACHE_MANAGE */

#if KMP_BARRIER_ICV_PUSH
      {
        KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
        if (propagate_icvs) {
          __kmp_init_implicit_task(team->t.t_ident,
                                   team->t.t_threads[child_tid], team,
                                   child_tid, FALSE);
          copy_icvs(&team->t.t_implicit_task_taskdata[child_tid].td_icvs,
                    &team->t.t_implicit_task_taskdata[0].td_icvs);
        }
      }
#endif // KMP_BARRIER_ICV_PUSH
      KA_TRACE(20,
               ("__kmp_tree_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)"
                "go(%p): %u => %u\n",
                gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
                child_bar->b_go + KMP_BARRIER_STATE_BUMP));
      // Release child from barrier
      ANNOTATE_BARRIER_BEGIN(child_thr);
      kmp_flag_64<> flag(&child_bar->b_go, child_thr);
      flag.release();
      child++;
      child_tid++;
    } while (child <= branch_factor && child_tid < nproc);
  }
  KA_TRACE(
      20, ("__kmp_tree_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
           gtid, team->t.t_id, tid, bt));
}

// Hyper Barrier
static void __kmp_hyper_barrier_gather(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hyper_gather);
  kmp_team_t *team = this_thr->th.th_team;
  kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
  kmp_info_t **other_threads = team->t.t_threads;
  kmp_uint64 new_state = KMP_BARRIER_UNUSED_STATE;
  kmp_uint32 num_threads = this_thr->th.th_team_nproc;
  kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt];
  kmp_uint32 branch_factor = 1 << branch_bits;
  kmp_uint32 offset;
  kmp_uint32 level;

  KA_TRACE(
      20,
      ("__kmp_hyper_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
       gtid, team->t.t_id, tid, bt));
  KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);

#if USE_ITT_BUILD && USE_ITT_NOTIFY
  // Barrier imbalance - save arrive time to the thread
  if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
    this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time =
        __itt_get_timestamp();
  }
#endif
  /* Perform a hypercube-embedded tree gather to wait until all of the threads
     have arrived, and reduce any required data as we go.  */
  kmp_flag_64<> p_flag(&thr_bar->b_arrived);
  for (level = 0, offset = 1; offset < num_threads;
       level += branch_bits, offset <<= branch_bits) {
    kmp_uint32 child;
    kmp_uint32 child_tid;

    if (((tid >> level) & (branch_factor - 1)) != 0) {
      kmp_int32 parent_tid = tid & ~((1 << (level + branch_bits)) - 1);

      KMP_MB(); // Synchronize parent and child threads.
      KA_TRACE(20,
               ("__kmp_hyper_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) "
                "arrived(%p): %llu => %llu\n",
                gtid, team->t.t_id, tid, __kmp_gtid_from_tid(parent_tid, team),
                team->t.t_id, parent_tid, &thr_bar->b_arrived,
                thr_bar->b_arrived,
                thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
      // Mark arrival to parent thread
      /* After performing this write (in the last iteration of the enclosing for
         loop), a worker thread may not assume that the team is valid any more
         - it could be deallocated by the primary thread at any time.  */
      ANNOTATE_BARRIER_BEGIN(this_thr);
      p_flag.set_waiter(other_threads[parent_tid]);
      p_flag.release();
      break;
    }

    // Parent threads wait for children to arrive
    if (new_state == KMP_BARRIER_UNUSED_STATE)
      new_state = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
    for (child = 1, child_tid = tid + (1 << level);
         child < branch_factor && child_tid < num_threads;
         child++, child_tid += (1 << level)) {
      kmp_info_t *child_thr = other_threads[child_tid];
      kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
#if KMP_CACHE_MANAGE
      kmp_uint32 next_child_tid = child_tid + (1 << level);
      // Prefetch next thread's arrived count
      if (child + 1 < branch_factor && next_child_tid < num_threads)
        KMP_CACHE_PREFETCH(
            &other_threads[next_child_tid]->th.th_bar[bt].bb.b_arrived);
#endif /* KMP_CACHE_MANAGE */
      KA_TRACE(20,
               ("__kmp_hyper_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) "
                "arrived(%p) == %llu\n",
                gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
      // Wait for child to arrive
      kmp_flag_64<> c_flag(&child_bar->b_arrived, new_state);
      c_flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
      ANNOTATE_BARRIER_END(child_thr);
      KMP_MB(); // Synchronize parent and child threads.
#if USE_ITT_BUILD && USE_ITT_NOTIFY
      // Barrier imbalance - write min of the thread time and a child time to
      // the thread.
      if (__kmp_forkjoin_frames_mode == 2) {
        this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
                                               child_thr->th.th_bar_min_time);
      }
#endif
      if (reduce) {
        KA_TRACE(100,
                 ("__kmp_hyper_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                  gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                  team->t.t_id, child_tid));
        ANNOTATE_REDUCE_AFTER(reduce);
        OMPT_REDUCTION_DECL(this_thr, gtid);
        OMPT_REDUCTION_BEGIN;
        (*reduce)(this_thr->th.th_local.reduce_data,
                  child_thr->th.th_local.reduce_data);
        OMPT_REDUCTION_END;
        ANNOTATE_REDUCE_BEFORE(reduce);
        ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
      }
    }
  }

  if (KMP_MASTER_TID(tid)) {
    // Need to update the team arrived pointer if we are the primary thread
    if (new_state == KMP_BARRIER_UNUSED_STATE)
      team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
    else
      team->t.t_bar[bt].b_arrived = new_state;
    KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) set team %d "
                  "arrived(%p) = %llu\n",
                  gtid, team->t.t_id, tid, team->t.t_id,
                  &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
  }
  KA_TRACE(
      20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
           gtid, team->t.t_id, tid, bt));
}

// The reverse versions seem to beat the forward versions overall
#define KMP_REVERSE_HYPER_BAR
static void __kmp_hyper_barrier_release(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hyper_release);
  kmp_team_t *team;
  kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
  kmp_info_t **other_threads;
  kmp_uint32 num_threads;
  kmp_uint32 branch_bits = __kmp_barrier_release_branch_bits[bt];
  kmp_uint32 branch_factor = 1 << branch_bits;
  kmp_uint32 child;
  kmp_uint32 child_tid;
  kmp_uint32 offset;
  kmp_uint32 level;

  /* Perform a hypercube-embedded tree release for all of the threads that have
     been gathered. If KMP_REVERSE_HYPER_BAR is defined (default) the threads
     are released in the reverse order of the corresponding gather, otherwise
     threads are released in the same order. */
  if (KMP_MASTER_TID(tid)) { // primary thread
    team = __kmp_threads[gtid]->th.th_team;
    KMP_DEBUG_ASSERT(team != NULL);
    KA_TRACE(20, ("__kmp_hyper_barrier_release: T#%d(%d:%d) primary enter for "
                  "barrier type %d\n",
                  gtid, team->t.t_id, tid, bt));
#if KMP_BARRIER_ICV_PUSH
    if (propagate_icvs) { // primary already has ICVs in final destination; copy
      copy_icvs(&thr_bar->th_fixed_icvs,
                &team->t.t_implicit_task_taskdata[tid].td_icvs);
    }
#endif
  } else { // Handle fork barrier workers who aren't part of a team yet
    KA_TRACE(20, ("__kmp_hyper_barrier_release: T#%d wait go(%p) == %u\n", gtid,
                  &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
    // Wait for parent thread to release us
    kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
    flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
    ANNOTATE_BARRIER_END(this_thr);
#if USE_ITT_BUILD && USE_ITT_NOTIFY
    if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
      // In fork barrier where we could not get the object reliably
      itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
      // Cancel wait on previous parallel region...
      __kmp_itt_task_starting(itt_sync_obj);

      if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
        return;

      itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
      if (itt_sync_obj != NULL)
        // Call prepare as early as possible for "new" barrier
        __kmp_itt_task_finished(itt_sync_obj);
    } else
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
        // Early exit for reaping threads releasing forkjoin barrier
        if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
      return;

    // The worker thread may now assume that the team is valid.
    team = __kmp_threads[gtid]->th.th_team;
    KMP_DEBUG_ASSERT(team != NULL);
    tid = __kmp_tid_from_gtid(gtid);

    TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
    KA_TRACE(20,
             ("__kmp_hyper_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
              gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
    KMP_MB(); // Flush all pending memory write invalidates.
  }
  num_threads = this_thr->th.th_team_nproc;
  other_threads = team->t.t_threads;

#ifdef KMP_REVERSE_HYPER_BAR
  // Count up to correct level for parent
  for (level = 0, offset = 1;
       offset < num_threads && (((tid >> level) & (branch_factor - 1)) == 0);
       level += branch_bits, offset <<= branch_bits)
    ;

  // Now go down from there
  for (level -= branch_bits, offset >>= branch_bits; offset != 0;
       level -= branch_bits, offset >>= branch_bits)
#else
  // Go down the tree, level by level
  for (level = 0, offset = 1; offset < num_threads;
       level += branch_bits, offset <<= branch_bits)
#endif // KMP_REVERSE_HYPER_BAR
  {
#ifdef KMP_REVERSE_HYPER_BAR
    /* Now go in reverse order through the children, highest to lowest.
       Initial setting of child is conservative here. */
    child = num_threads >> ((level == 0) ? level : level - 1);
    for (child = (child < branch_factor - 1) ? child : branch_factor - 1,
        child_tid = tid + (child << level);
         child >= 1; child--, child_tid -= (1 << level))
#else
    if (((tid >> level) & (branch_factor - 1)) != 0)
      // No need to go lower than this, since this is the level parent would be
      // notified
      break;
    // Iterate through children on this level of the tree
    for (child = 1, child_tid = tid + (1 << level);
         child < branch_factor && child_tid < num_threads;
         child++, child_tid += (1 << level))
#endif // KMP_REVERSE_HYPER_BAR
    {
      if (child_tid >= num_threads)
        continue; // Child doesn't exist so keep going
      else {
        kmp_info_t *child_thr = other_threads[child_tid];
        kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
#if KMP_CACHE_MANAGE
        kmp_uint32 next_child_tid = child_tid - (1 << level);
// Prefetch next thread's go count
#ifdef KMP_REVERSE_HYPER_BAR
        if (child - 1 >= 1 && next_child_tid < num_threads)
#else
        if (child + 1 < branch_factor && next_child_tid < num_threads)
#endif // KMP_REVERSE_HYPER_BAR
          KMP_CACHE_PREFETCH(
              &other_threads[next_child_tid]->th.th_bar[bt].bb.b_go);
#endif /* KMP_CACHE_MANAGE */

#if KMP_BARRIER_ICV_PUSH
        if (propagate_icvs) // push my fixed ICVs to my child
          copy_icvs(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
#endif // KMP_BARRIER_ICV_PUSH

        KA_TRACE(
            20,
            ("__kmp_hyper_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)"
             "go(%p): %u => %u\n",
             gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
             team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
             child_bar->b_go + KMP_BARRIER_STATE_BUMP));
        // Release child from barrier
        ANNOTATE_BARRIER_BEGIN(child_thr);
        kmp_flag_64<> flag(&child_bar->b_go, child_thr);
        flag.release();
      }
    }
  }
#if KMP_BARRIER_ICV_PUSH
  if (propagate_icvs &&
      !KMP_MASTER_TID(tid)) { // copy ICVs locally to final dest
    __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team, tid,
                             FALSE);
    copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
              &thr_bar->th_fixed_icvs);
  }
#endif
  KA_TRACE(
      20,
      ("__kmp_hyper_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
       gtid, team->t.t_id, tid, bt));
}

// Hierarchical Barrier

// Initialize thread barrier data
/* Initializes/re-initializes the hierarchical barrier data stored on a thread.
   Performs the minimum amount of initialization required based on how the team
   has changed. Returns true if leaf children will require both on-core and
   traditional wake-up mechanisms. For example, if the team size increases,
   threads already in the team will respond to on-core wakeup on their parent
   thread, but threads newly added to the team will only be listening on the
   their local b_go. */
static bool __kmp_init_hierarchical_barrier_thread(enum barrier_type bt,
                                                   kmp_bstate_t *thr_bar,
                                                   kmp_uint32 nproc, int gtid,
                                                   int tid, kmp_team_t *team) {
  // Checks to determine if (re-)initialization is needed
  bool uninitialized = thr_bar->team == NULL;
  bool team_changed = team != thr_bar->team;
  bool team_sz_changed = nproc != thr_bar->nproc;
  bool tid_changed = tid != thr_bar->old_tid;
  bool retval = false;

  if (uninitialized || team_sz_changed) {
    __kmp_get_hierarchy(nproc, thr_bar);
  }

  if (uninitialized || team_sz_changed || tid_changed) {
    thr_bar->my_level = thr_bar->depth - 1; // default for primary thread
    thr_bar->parent_tid = -1; // default for primary thread
    if (!KMP_MASTER_TID(tid)) {
      // if not primary thread, find parent thread in hierarchy
      kmp_uint32 d = 0;
      while (d < thr_bar->depth) { // find parent based on level of thread in
        // hierarchy, and note level
        kmp_uint32 rem;
        if (d == thr_bar->depth - 2) { // reached level right below the primary
          thr_bar->parent_tid = 0;
          thr_bar->my_level = d;
          break;
        } else if ((rem = tid % thr_bar->skip_per_level[d + 1]) != 0) {
          // TODO: can we make the above op faster?
          // thread is not a subtree root at next level, so this is max
          thr_bar->parent_tid = tid - rem;
          thr_bar->my_level = d;
          break;
        }
        ++d;
      }
    }
    __kmp_type_convert(7 - ((tid - thr_bar->parent_tid) /
                            (thr_bar->skip_per_level[thr_bar->my_level])),
                       &(thr_bar->offset));
    thr_bar->old_tid = tid;
    thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
    thr_bar->team = team;
    thr_bar->parent_bar =
        &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
  }
  if (uninitialized || team_changed || tid_changed) {
    thr_bar->team = team;
    thr_bar->parent_bar =
        &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
    retval = true;
  }
  if (uninitialized || team_sz_changed || tid_changed) {
    thr_bar->nproc = nproc;
    thr_bar->leaf_kids = thr_bar->base_leaf_kids;
    if (thr_bar->my_level == 0)
      thr_bar->leaf_kids = 0;
    if (thr_bar->leaf_kids && (kmp_uint32)tid + thr_bar->leaf_kids + 1 > nproc)
      __kmp_type_convert(nproc - tid - 1, &(thr_bar->leaf_kids));
    thr_bar->leaf_state = 0;
    for (int i = 0; i < thr_bar->leaf_kids; ++i)
      ((char *)&(thr_bar->leaf_state))[7 - i] = 1;
  }
  return retval;
}

static void __kmp_hierarchical_barrier_gather(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hier_gather);
  kmp_team_t *team = this_thr->th.th_team;
  kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
  kmp_uint32 nproc = this_thr->th.th_team_nproc;
  kmp_info_t **other_threads = team->t.t_threads;
  kmp_uint64 new_state = 0;

  int level = team->t.t_level;
  if (other_threads[0]
          ->th.th_teams_microtask) // are we inside the teams construct?
    if (this_thr->th.th_teams_size.nteams > 1)
      ++level; // level was not increased in teams construct for team_of_masters
  if (level == 1)
    thr_bar->use_oncore_barrier = 1;
  else
    thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested

  KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) enter for "
                "barrier type %d\n",
                gtid, team->t.t_id, tid, bt));
  KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);

#if USE_ITT_BUILD && USE_ITT_NOTIFY
  // Barrier imbalance - save arrive time to the thread
  if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
    this_thr->th.th_bar_arrive_time = __itt_get_timestamp();
  }
#endif

  (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid,
                                               team);

  if (thr_bar->my_level) { // not a leaf (my_level==0 means leaf)
    kmp_int32 child_tid;
    new_state =
        (kmp_uint64)team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
    if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
        thr_bar->use_oncore_barrier) {
      if (thr_bar->leaf_kids) {
        // First, wait for leaf children to check-in on my b_arrived flag
        kmp_uint64 leaf_state =
            KMP_MASTER_TID(tid)
                ? thr_bar->b_arrived | thr_bar->leaf_state
                : team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;
        KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) waiting "
                      "for leaf kids\n",
                      gtid, team->t.t_id, tid));
        kmp_flag_64<> flag(&thr_bar->b_arrived, leaf_state);
        flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
        if (reduce) {
          ANNOTATE_REDUCE_AFTER(reduce);
          OMPT_REDUCTION_DECL(this_thr, gtid);
          OMPT_REDUCTION_BEGIN;
          for (child_tid = tid + 1; child_tid <= tid + thr_bar->leaf_kids;
               ++child_tid) {
            KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += "
                           "T#%d(%d:%d)\n",
                           gtid, team->t.t_id, tid,
                           __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                           child_tid));
            ANNOTATE_BARRIER_END(other_threads[child_tid]);
            (*reduce)(this_thr->th.th_local.reduce_data,
                      other_threads[child_tid]->th.th_local.reduce_data);
          }
          OMPT_REDUCTION_END;
          ANNOTATE_REDUCE_BEFORE(reduce);
          ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
        }
        // clear leaf_state bits
        KMP_TEST_THEN_AND64(&thr_bar->b_arrived, ~(thr_bar->leaf_state));
      }
      // Next, wait for higher level children on each child's b_arrived flag
      for (kmp_uint32 d = 1; d < thr_bar->my_level;
           ++d) { // gather lowest level threads first, but skip 0
        kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
                   skip = thr_bar->skip_per_level[d];
        if (last > nproc)
          last = nproc;
        for (child_tid = tid + skip; child_tid < (int)last; child_tid += skip) {
          kmp_info_t *child_thr = other_threads[child_tid];
          kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
          KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait "
                        "T#%d(%d:%d) "
                        "arrived(%p) == %llu\n",
                        gtid, team->t.t_id, tid,
                        __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                        child_tid, &child_bar->b_arrived, new_state));
          kmp_flag_64<> flag(&child_bar->b_arrived, new_state);
          flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
          ANNOTATE_BARRIER_END(child_thr);
          if (reduce) {
            KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += "
                           "T#%d(%d:%d)\n",
                           gtid, team->t.t_id, tid,
                           __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                           child_tid));
            ANNOTATE_REDUCE_AFTER(reduce);
            (*reduce)(this_thr->th.th_local.reduce_data,
                      child_thr->th.th_local.reduce_data);
            ANNOTATE_REDUCE_BEFORE(reduce);
            ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
          }
        }
      }
    } else { // Blocktime is not infinite
      for (kmp_uint32 d = 0; d < thr_bar->my_level;
           ++d) { // Gather lowest level threads first
        kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
                   skip = thr_bar->skip_per_level[d];
        if (last > nproc)
          last = nproc;
        for (child_tid = tid + skip; child_tid < (int)last; child_tid += skip) {
          kmp_info_t *child_thr = other_threads[child_tid];
          kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
          KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait "
                        "T#%d(%d:%d) "
                        "arrived(%p) == %llu\n",
                        gtid, team->t.t_id, tid,
                        __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                        child_tid, &child_bar->b_arrived, new_state));
          kmp_flag_64<> flag(&child_bar->b_arrived, new_state);
          flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
          ANNOTATE_BARRIER_END(child_thr);
          if (reduce) {
            KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += "
                           "T#%d(%d:%d)\n",
                           gtid, team->t.t_id, tid,
                           __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                           child_tid));
            ANNOTATE_REDUCE_AFTER(reduce);
            (*reduce)(this_thr->th.th_local.reduce_data,
                      child_thr->th.th_local.reduce_data);
            ANNOTATE_REDUCE_BEFORE(reduce);
            ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
          }
        }
      }
    }
  }
  // All subordinates are gathered; now release parent if not primary thread

  if (!KMP_MASTER_TID(tid)) { // worker threads release parent in hierarchy
    KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) releasing"
                  " T#%d(%d:%d) arrived(%p): %llu => %llu\n",
                  gtid, team->t.t_id, tid,
                  __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id,
                  thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
                  thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
    /* Mark arrival to parent: After performing this write, a worker thread may
       not assume that the team is valid any more - it could be deallocated by
       the primary thread at any time. */
    if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ||
        !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived
      // flag; release it
      ANNOTATE_BARRIER_BEGIN(this_thr);
      kmp_flag_64<> flag(&thr_bar->b_arrived,
                         other_threads[thr_bar->parent_tid]);
      flag.release();
    } else {
      // Leaf does special release on "offset" bits of parent's b_arrived flag
      thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
      kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived,
                           thr_bar->offset + 1);
      flag.set_waiter(other_threads[thr_bar->parent_tid]);
      flag.release();
    }
  } else { // Primary thread needs to update the team's b_arrived value
    team->t.t_bar[bt].b_arrived = new_state;
    KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) set team %d "
                  "arrived(%p) = %llu\n",
                  gtid, team->t.t_id, tid, team->t.t_id,
                  &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
  }
  // Is the team access below unsafe or just technically invalid?
  KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) exit for "
                "barrier type %d\n",
                gtid, team->t.t_id, tid, bt));
}

static void __kmp_hierarchical_barrier_release(
    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
    int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hier_release);
  kmp_team_t *team;
  kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
  kmp_uint32 nproc;
  bool team_change = false; // indicates on-core barrier shouldn't be used

  if (KMP_MASTER_TID(tid)) {
    team = __kmp_threads[gtid]->th.th_team;
    KMP_DEBUG_ASSERT(team != NULL);
    KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) primary "
                  "entered barrier type %d\n",
                  gtid, team->t.t_id, tid, bt));
  } else { // Worker threads
    // Wait for parent thread to release me
    if (!thr_bar->use_oncore_barrier ||
        __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || thr_bar->my_level != 0 ||
        thr_bar->team == NULL) {
      // Use traditional method of waiting on my own b_go flag
      thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG;
      kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
      flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
      ANNOTATE_BARRIER_END(this_thr);
      TCW_8(thr_bar->b_go,
            KMP_INIT_BARRIER_STATE); // Reset my b_go flag for next time
    } else { // Thread barrier data is initialized, this is a leaf, blocktime is
      // infinite, not nested
      // Wait on my "offset" bits on parent's b_go flag
      thr_bar->wait_flag = KMP_BARRIER_PARENT_FLAG;
      kmp_flag_oncore flag(&thr_bar->parent_bar->b_go, KMP_BARRIER_STATE_BUMP,
                           thr_bar->offset + 1, bt,
                           this_thr USE_ITT_BUILD_ARG(itt_sync_obj));
      flag.wait(this_thr, TRUE);
      if (thr_bar->wait_flag ==
          KMP_BARRIER_SWITCHING) { // Thread was switched to own b_go
        TCW_8(thr_bar->b_go,
              KMP_INIT_BARRIER_STATE); // Reset my b_go flag for next time
      } else { // Reset my bits on parent's b_go flag
        (RCAST(volatile char *,
               &(thr_bar->parent_bar->b_go)))[thr_bar->offset + 1] = 0;
      }
    }
    thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
    // Early exit for reaping threads releasing forkjoin barrier
    if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
      return;
    // The worker thread may now assume that the team is valid.
    team = __kmp_threads[gtid]->th.th_team;
    KMP_DEBUG_ASSERT(team != NULL);
    tid = __kmp_tid_from_gtid(gtid);

    KA_TRACE(
        20,
        ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
         gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
    KMP_MB(); // Flush all pending memory write invalidates.
  }

  nproc = this_thr->th.th_team_nproc;
  int level = team->t.t_level;
  if (team->t.t_threads[0]
          ->th.th_teams_microtask) { // are we inside the teams construct?
    if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
        this_thr->th.th_teams_level == level)
      ++level; // level was not increased in teams construct for team_of_workers
    if (this_thr->th.th_teams_size.nteams > 1)
      ++level; // level was not increased in teams construct for team_of_masters
  }
  if (level == 1)
    thr_bar->use_oncore_barrier = 1;
  else
    thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested

  // If the team size has increased, we still communicate with old leaves via
  // oncore barrier.
  unsigned short int old_leaf_kids = thr_bar->leaf_kids;
  kmp_uint64 old_leaf_state = thr_bar->leaf_state;
  team_change = __kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid,
                                                       tid, team);
  // But if the entire team changes, we won't use oncore barrier at all
  if (team_change)
    old_leaf_kids = 0;

#if KMP_BARRIER_ICV_PUSH
  if (propagate_icvs) {
    __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team, tid,
                             FALSE);
    if (KMP_MASTER_TID(
            tid)) { // primary already has copy in final destination; copy
      copy_icvs(&thr_bar->th_fixed_icvs,
                &team->t.t_implicit_task_taskdata[tid].td_icvs);
    } else if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
               thr_bar->use_oncore_barrier) { // optimization for inf blocktime
      if (!thr_bar->my_level) // I'm a leaf in the hierarchy (my_level==0)
        // leaves (on-core children) pull parent's fixed ICVs directly to local
        // ICV store
        copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
                  &thr_bar->parent_bar->th_fixed_icvs);
      // non-leaves will get ICVs piggybacked with b_go via NGO store
    } else { // blocktime is not infinite; pull ICVs from parent's fixed ICVs
      if (thr_bar->my_level) // not a leaf; copy ICVs to my fixed ICVs child can
        // access
        copy_icvs(&thr_bar->th_fixed_icvs, &thr_bar->parent_bar->th_fixed_icvs);
      else // leaves copy parent's fixed ICVs directly to local ICV store
        copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
                  &thr_bar->parent_bar->th_fixed_icvs);
    }
  }
#endif // KMP_BARRIER_ICV_PUSH

  // Now, release my children
  if (thr_bar->my_level) { // not a leaf
    kmp_int32 child_tid;
    kmp_uint32 last;
    if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
        thr_bar->use_oncore_barrier) {
      if (KMP_MASTER_TID(tid)) { // do a flat release
        // Set local b_go to bump children via NGO store of the cache line
        // containing IVCs and b_go.
        thr_bar->b_go = KMP_BARRIER_STATE_BUMP;
        // Use ngo stores if available; b_go piggybacks in the last 8 bytes of
        // the cache line
        ngo_load(&thr_bar->th_fixed_icvs);
        // This loops over all the threads skipping only the leaf nodes in the
        // hierarchy
        for (child_tid = thr_bar->skip_per_level[1]; child_tid < (int)nproc;
             child_tid += thr_bar->skip_per_level[1]) {
          kmp_bstate_t *child_bar =
              &team->t.t_threads[child_tid]->th.th_bar[bt].bb;
          KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) "
                        "releasing T#%d(%d:%d)"
                        " go(%p): %u => %u\n",
                        gtid, team->t.t_id, tid,
                        __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                        child_tid, &child_bar->b_go, child_bar->b_go,
                        child_bar->b_go + KMP_BARRIER_STATE_BUMP));
          // Use ngo store (if available) to both store ICVs and release child
          // via child's b_go
          ngo_store_go(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
        }
        ngo_sync();
      }
      TCW_8(thr_bar->b_go,
            KMP_INIT_BARRIER_STATE); // Reset my b_go flag for next time
      // Now, release leaf children
      if (thr_bar->leaf_kids) { // if there are any
        // We test team_change on the off-chance that the level 1 team changed.
        if (team_change ||
            old_leaf_kids < thr_bar->leaf_kids) { // some old, some new
          if (old_leaf_kids) { // release old leaf kids
            thr_bar->b_go |= old_leaf_state;
          }
          // Release new leaf kids
          last = tid + thr_bar->skip_per_level[1];
          if (last > nproc)
            last = nproc;
          for (child_tid = tid + 1 + old_leaf_kids; child_tid < (int)last;
               ++child_tid) { // skip_per_level[0]=1
            kmp_info_t *child_thr = team->t.t_threads[child_tid];
            kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
            KA_TRACE(
                20,
                ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) releasing"
                 " T#%d(%d:%d) go(%p): %u => %u\n",
                 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                 team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
                 child_bar->b_go + KMP_BARRIER_STATE_BUMP));
            // Release child using child's b_go flag
            ANNOTATE_BARRIER_BEGIN(child_thr);
            kmp_flag_64<> flag(&child_bar->b_go, child_thr);
            flag.release();
          }
        } else { // Release all children at once with leaf_state bits on my own
          // b_go flag
          thr_bar->b_go |= thr_bar->leaf_state;
        }
      }
    } else { // Blocktime is not infinite; do a simple hierarchical release
      for (int d = thr_bar->my_level - 1; d >= 0;
           --d) { // Release highest level threads first
        last = tid + thr_bar->skip_per_level[d + 1];
        kmp_uint32 skip = thr_bar->skip_per_level[d];
        if (last > nproc)
          last = nproc;
        for (child_tid = tid + skip; child_tid < (int)last; child_tid += skip) {
          kmp_info_t *child_thr = team->t.t_threads[child_tid];
          kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
          KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) "
                        "releasing T#%d(%d:%d) go(%p): %u => %u\n",
                        gtid, team->t.t_id, tid,
                        __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                        child_tid, &child_bar->b_go, child_bar->b_go,
                        child_bar->b_go + KMP_BARRIER_STATE_BUMP));
          // Release child using child's b_go flag
          ANNOTATE_BARRIER_BEGIN(child_thr);
          kmp_flag_64<> flag(&child_bar->b_go, child_thr);
          flag.release();
        }
      }
    }
#if KMP_BARRIER_ICV_PUSH
    if (propagate_icvs && !KMP_MASTER_TID(tid))
      // non-leaves copy ICVs from fixed ICVs to local dest
      copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
                &thr_bar->th_fixed_icvs);
#endif // KMP_BARRIER_ICV_PUSH
  }
  KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) exit for "
                "barrier type %d\n",
                gtid, team->t.t_id, tid, bt));
}

// End of Barrier Algorithms

// type traits for cancellable value
// if cancellable is true, then is_cancellable is a normal boolean variable
// if cancellable is false, then is_cancellable is a compile time constant
template <bool cancellable> struct is_cancellable {};
template <> struct is_cancellable<true> {
  bool value;
  is_cancellable() : value(false) {}
  is_cancellable(bool b) : value(b) {}
  is_cancellable &operator=(bool b) {
    value = b;
    return *this;
  }
  operator bool() const { return value; }
};
template <> struct is_cancellable<false> {
  is_cancellable &operator=(bool b) { return *this; }
  constexpr operator bool() const { return false; }
};

// Internal function to do a barrier.
/* If is_split is true, do a split barrier, otherwise, do a plain barrier
   If reduce is non-NULL, do a split reduction barrier, otherwise, do a split
   barrier
   When cancellable = false,
     Returns 0 if primary thread, 1 if worker thread.
   When cancellable = true
     Returns 0 if not cancelled, 1 if cancelled.  */
template <bool cancellable = false>
static int __kmp_barrier_template(enum barrier_type bt, int gtid, int is_split,
                                  size_t reduce_size, void *reduce_data,
                                  void (*reduce)(void *, void *)) {
  KMP_TIME_PARTITIONED_BLOCK(OMP_plain_barrier);
  KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
  int tid = __kmp_tid_from_gtid(gtid);
  kmp_info_t *this_thr = __kmp_threads[gtid];
  kmp_team_t *team = this_thr->th.th_team;
  int status = 0;
  is_cancellable<cancellable> cancelled;
#if OMPT_SUPPORT && OMPT_OPTIONAL
  ompt_data_t *my_task_data;
  ompt_data_t *my_parallel_data;
  void *return_address;
  ompt_sync_region_t barrier_kind;
#endif

  KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) has arrived\n", gtid,
                __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid)));

  ANNOTATE_BARRIER_BEGIN(&team->t.t_bar);
#if OMPT_SUPPORT
  if (ompt_enabled.enabled) {
#if OMPT_OPTIONAL
    my_task_data = OMPT_CUR_TASK_DATA(this_thr);
    my_parallel_data = OMPT_CUR_TEAM_DATA(this_thr);
    return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
    barrier_kind = __ompt_get_barrier_kind(bt, this_thr);
    if (ompt_enabled.ompt_callback_sync_region) {
      ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
          barrier_kind, ompt_scope_begin, my_parallel_data, my_task_data,
          return_address);
    }
    if (ompt_enabled.ompt_callback_sync_region_wait) {
      ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
          barrier_kind, ompt_scope_begin, my_parallel_data, my_task_data,
          return_address);
    }
#endif
    // It is OK to report the barrier state after the barrier begin callback.
    // According to the OMPT specification, a compliant implementation may
    // even delay reporting this state until the barrier begins to wait.
    this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier;
  }
#endif

  if (!team->t.t_serialized) {
#if USE_ITT_BUILD
    // This value will be used in itt notify events below.
    void *itt_sync_obj = NULL;
#if USE_ITT_NOTIFY
    if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
      itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1);
#endif
#endif /* USE_ITT_BUILD */
    if (__kmp_tasking_mode == tskm_extra_barrier) {
      __kmp_tasking_barrier(team, this_thr, gtid);
      KA_TRACE(15,
               ("__kmp_barrier: T#%d(%d:%d) past tasking barrier\n", gtid,
                __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid)));
    }

    /* Copy the blocktime info to the thread, where __kmp_wait_template() can
       access it when the team struct is not guaranteed to exist. */
    // See note about the corresponding code in __kmp_join_barrier() being
    // performance-critical.
    if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
#if KMP_USE_MONITOR
      this_thr->th.th_team_bt_intervals =
          team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
      this_thr->th.th_team_bt_set =
          team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
#else
      this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid);
#endif
    }

#if USE_ITT_BUILD
    if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
      __kmp_itt_barrier_starting(gtid, itt_sync_obj);
#endif /* USE_ITT_BUILD */
#if USE_DEBUGGER
    // Let the debugger know: the thread arrived to the barrier and waiting.
    if (KMP_MASTER_TID(tid)) { // Primary thread counter stored in team struct
      team->t.t_bar[bt].b_master_arrived += 1;
    } else {
      this_thr->th.th_bar[bt].bb.b_worker_arrived += 1;
    } // if
#endif /* USE_DEBUGGER */
    if (reduce != NULL) {
      // KMP_DEBUG_ASSERT( is_split == TRUE );  // #C69956
      this_thr->th.th_local.reduce_data = reduce_data;
    }

    if (KMP_MASTER_TID(tid) && __kmp_tasking_mode != tskm_immediate_exec)
      // use 0 to only setup the current team if nthreads > 1
      __kmp_task_team_setup(this_thr, team, 0);

    if (cancellable) {
      cancelled = __kmp_linear_barrier_gather_cancellable(
          bt, this_thr, gtid, tid, reduce USE_ITT_BUILD_ARG(itt_sync_obj));
    } else {
      switch (__kmp_barrier_gather_pattern[bt]) {
      case bp_dist_bar: {
        __kmp_dist_barrier_gather(bt, this_thr, gtid, tid,
                                  reduce USE_ITT_BUILD_ARG(itt_sync_obj));
        break;
      }
      case bp_hyper_bar: {
        // don't set branch bits to 0; use linear
        KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]);
        __kmp_hyper_barrier_gather(bt, this_thr, gtid, tid,
                                   reduce USE_ITT_BUILD_ARG(itt_sync_obj));
        break;
      }
      case bp_hierarchical_bar: {
        __kmp_hierarchical_barrier_gather(
            bt, this_thr, gtid, tid, reduce USE_ITT_BUILD_ARG(itt_sync_obj));
        break;
      }
      case bp_tree_bar: {
        // don't set branch bits to 0; use linear
        KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]);
        __kmp_tree_barrier_gather(bt, this_thr, gtid, tid,
                                  reduce USE_ITT_BUILD_ARG(itt_sync_obj));
        break;
      }
      default: {
        __kmp_linear_barrier_gather(bt, this_thr, gtid, tid,
                                    reduce USE_ITT_BUILD_ARG(itt_sync_obj));
      }
      }
    }

    KMP_MB();

    if (KMP_MASTER_TID(tid)) {
      status = 0;
      if (__kmp_tasking_mode != tskm_immediate_exec && !cancelled) {
        __kmp_task_team_wait(this_thr, team USE_ITT_BUILD_ARG(itt_sync_obj));
      }
#if USE_DEBUGGER
      // Let the debugger know: All threads are arrived and starting leaving the
      // barrier.
      team->t.t_bar[bt].b_team_arrived += 1;
#endif

      if (__kmp_omp_cancellation) {
        kmp_int32 cancel_request = KMP_ATOMIC_LD_RLX(&team->t.t_cancel_request);
        // Reset cancellation flag for worksharing constructs
        if (cancel_request == cancel_loop ||
            cancel_request == cancel_sections) {
          KMP_ATOMIC_ST_RLX(&team->t.t_cancel_request, cancel_noreq);
        }
      }
#if USE_ITT_BUILD
      /* TODO: In case of split reduction barrier, primary thread may send
         acquired event early, before the final summation into the shared
         variable is done (final summation can be a long operation for array
         reductions).  */
      if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
        __kmp_itt_barrier_middle(gtid, itt_sync_obj);
#endif /* USE_ITT_BUILD */
#if USE_ITT_BUILD && USE_ITT_NOTIFY
      // Barrier - report frame end (only if active_level == 1)
      if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
          __kmp_forkjoin_frames_mode &&
          (this_thr->th.th_teams_microtask == NULL || // either not in teams
           this_thr->th.th_teams_size.nteams == 1) && // or inside single team
          team->t.t_active_level == 1) {
        ident_t *loc = __kmp_threads[gtid]->th.th_ident;
        kmp_uint64 cur_time = __itt_get_timestamp();
        kmp_info_t **other_threads = team->t.t_threads;
        int nproc = this_thr->th.th_team_nproc;
        int i;
        switch (__kmp_forkjoin_frames_mode) {
        case 1:
          __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0,
                                 loc, nproc);
          this_thr->th.th_frame_time = cur_time;
          break;
        case 2: // AC 2015-01-19: currently does not work for hierarchical (to
          // be fixed)
          __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time,
                                 1, loc, nproc);
          break;
        case 3:
          if (__itt_metadata_add_ptr) {
            // Initialize with primary thread's wait time
            kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
            // Set arrive time to zero to be able to check it in
            // __kmp_invoke_task(); the same is done inside the loop below
            this_thr->th.th_bar_arrive_time = 0;
            for (i = 1; i < nproc; ++i) {
              delta += (cur_time - other_threads[i]->th.th_bar_arrive_time);
              other_threads[i]->th.th_bar_arrive_time = 0;
            }
            __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time,
                                         cur_time, delta,
                                         (kmp_uint64)(reduce != NULL));
          }
          __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0,
                                 loc, nproc);
          this_thr->th.th_frame_time = cur_time;
          break;
        }
      }
#endif /* USE_ITT_BUILD */
    } else {
      status = 1;
#if USE_ITT_BUILD
      if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
        __kmp_itt_barrier_middle(gtid, itt_sync_obj);
#endif /* USE_ITT_BUILD */
    }
    if ((status == 1 || !is_split) && !cancelled) {
      if (cancellable) {
        cancelled = __kmp_linear_barrier_release_cancellable(
            bt, this_thr, gtid, tid, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
      } else {
        switch (__kmp_barrier_release_pattern[bt]) {
        case bp_dist_bar: {
          KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
          __kmp_dist_barrier_release(bt, this_thr, gtid, tid,
                                     FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
          break;
        }
        case bp_hyper_bar: {
          KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
          __kmp_hyper_barrier_release(bt, this_thr, gtid, tid,
                                      FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
          break;
        }
        case bp_hierarchical_bar: {
          __kmp_hierarchical_barrier_release(
              bt, this_thr, gtid, tid, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
          break;
        }
        case bp_tree_bar: {
          KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
          __kmp_tree_barrier_release(bt, this_thr, gtid, tid,
                                     FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
          break;
        }
        default: {
          __kmp_linear_barrier_release(bt, this_thr, gtid, tid,
                                       FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
        }
        }
      }
      if (__kmp_tasking_mode != tskm_immediate_exec && !cancelled) {
        __kmp_task_team_sync(this_thr, team);
      }
    }

#if USE_ITT_BUILD
    /* GEH: TODO: Move this under if-condition above and also include in
       __kmp_end_split_barrier(). This will more accurately represent the actual
       release time of the threads for split barriers.  */
    if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
      __kmp_itt_barrier_finished(gtid, itt_sync_obj);
#endif /* USE_ITT_BUILD */
  } else { // Team is serialized.
    status = 0;
    if (__kmp_tasking_mode != tskm_immediate_exec) {
      if (this_thr->th.th_task_team != NULL) {
#if USE_ITT_NOTIFY
        void *itt_sync_obj = NULL;
        if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
          itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1);
          __kmp_itt_barrier_starting(gtid, itt_sync_obj);
        }
#endif

        KMP_DEBUG_ASSERT(this_thr->th.th_task_team->tt.tt_found_proxy_tasks ==
                         TRUE);
        __kmp_task_team_wait(this_thr, team USE_ITT_BUILD_ARG(itt_sync_obj));
        __kmp_task_team_setup(this_thr, team, 0);

#if USE_ITT_BUILD
        if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
          __kmp_itt_barrier_finished(gtid, itt_sync_obj);
#endif /* USE_ITT_BUILD */
      }
    }
  }
  KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) is leaving with return value %d\n",
                gtid, __kmp_team_from_gtid(gtid)->t.t_id,
                __kmp_tid_from_gtid(gtid), status));

#if OMPT_SUPPORT
  if (ompt_enabled.enabled) {
#if OMPT_OPTIONAL
    if (ompt_enabled.ompt_callback_sync_region_wait) {
      ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
          barrier_kind, ompt_scope_end, my_parallel_data, my_task_data,
          return_address);
    }
    if (ompt_enabled.ompt_callback_sync_region) {
      ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
          barrier_kind, ompt_scope_end, my_parallel_data, my_task_data,
          return_address);
    }
#endif
    this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
  }
#endif
  ANNOTATE_BARRIER_END(&team->t.t_bar);

  if (cancellable)
    return (int)cancelled;
  return status;
}

// Returns 0 if primary thread, 1 if worker thread.
int __kmp_barrier(enum barrier_type bt, int gtid, int is_split,
                  size_t reduce_size, void *reduce_data,
                  void (*reduce)(void *, void *)) {
  return __kmp_barrier_template<>(bt, gtid, is_split, reduce_size, reduce_data,
                                  reduce);
}

#if defined(KMP_GOMP_COMPAT)
// Returns 1 if cancelled, 0 otherwise
int __kmp_barrier_gomp_cancel(int gtid) {
  if (__kmp_omp_cancellation) {
    int cancelled = __kmp_barrier_template<true>(bs_plain_barrier, gtid, FALSE,
                                                 0, NULL, NULL);
    if (cancelled) {
      int tid = __kmp_tid_from_gtid(gtid);
      kmp_info_t *this_thr = __kmp_threads[gtid];
      if (KMP_MASTER_TID(tid)) {
        // Primary thread does not need to revert anything
      } else {
        // Workers need to revert their private b_arrived flag
        this_thr->th.th_bar[bs_plain_barrier].bb.b_arrived -=
            KMP_BARRIER_STATE_BUMP;
      }
    }
    return cancelled;
  }
  __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
  return FALSE;
}
#endif

void __kmp_end_split_barrier(enum barrier_type bt, int gtid) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_end_split_barrier);
  KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
  KMP_DEBUG_ASSERT(bt < bs_last_barrier);
  int tid = __kmp_tid_from_gtid(gtid);
  kmp_info_t *this_thr = __kmp_threads[gtid];
  kmp_team_t *team = this_thr->th.th_team;

  ANNOTATE_BARRIER_BEGIN(&team->t.t_bar);
  if (!team->t.t_serialized) {
    if (KMP_MASTER_GTID(gtid)) {
      switch (__kmp_barrier_release_pattern[bt]) {
      case bp_dist_bar: {
        __kmp_dist_barrier_release(bt, this_thr, gtid, tid,
                                   FALSE USE_ITT_BUILD_ARG(NULL));
        break;
      }
      case bp_hyper_bar: {
        KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
        __kmp_hyper_barrier_release(bt, this_thr, gtid, tid,
                                    FALSE USE_ITT_BUILD_ARG(NULL));
        break;
      }
      case bp_hierarchical_bar: {
        __kmp_hierarchical_barrier_release(bt, this_thr, gtid, tid,
                                           FALSE USE_ITT_BUILD_ARG(NULL));
        break;
      }
      case bp_tree_bar: {
        KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
        __kmp_tree_barrier_release(bt, this_thr, gtid, tid,
                                   FALSE USE_ITT_BUILD_ARG(NULL));
        break;
      }
      default: {
        __kmp_linear_barrier_release(bt, this_thr, gtid, tid,
                                     FALSE USE_ITT_BUILD_ARG(NULL));
      }
      }
      if (__kmp_tasking_mode != tskm_immediate_exec) {
        __kmp_task_team_sync(this_thr, team);
      } // if
    }
  }
  ANNOTATE_BARRIER_END(&team->t.t_bar);
}

void __kmp_join_barrier(int gtid) {
  KMP_TIME_PARTITIONED_BLOCK(OMP_join_barrier);
  KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER);

  KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);

  kmp_info_t *this_thr = __kmp_threads[gtid];
  kmp_team_t *team;
  kmp_uint nproc;
  kmp_info_t *master_thread;
  int tid;
#ifdef KMP_DEBUG
  int team_id;
#endif /* KMP_DEBUG */
#if USE_ITT_BUILD
  void *itt_sync_obj = NULL;
#if USE_ITT_NOTIFY
  if (__itt_sync_create_ptr || KMP_ITT_DEBUG) // Don't call routine without need
    // Get object created at fork_barrier
    itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
#endif
#endif /* USE_ITT_BUILD */
  KMP_MB();

  // Get current info
  team = this_thr->th.th_team;
  nproc = this_thr->th.th_team_nproc;
  KMP_DEBUG_ASSERT((int)nproc == team->t.t_nproc);
  tid = __kmp_tid_from_gtid(gtid);
#ifdef KMP_DEBUG
  team_id = team->t.t_id;
#endif /* KMP_DEBUG */
  master_thread = this_thr->th.th_team_master;
#ifdef KMP_DEBUG
  if (master_thread != team->t.t_threads[0]) {
    __kmp_print_structure();
  }
#endif /* KMP_DEBUG */
  KMP_DEBUG_ASSERT(master_thread == team->t.t_threads[0]);
  KMP_MB();

  // Verify state
  KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_team));
  KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_root));
  KMP_DEBUG_ASSERT(this_thr == team->t.t_threads[tid]);
  KA_TRACE(10, ("__kmp_join_barrier: T#%d(%d:%d) arrived at join barrier\n",
                gtid, team_id, tid));

  ANNOTATE_BARRIER_BEGIN(&team->t.t_bar);
#if OMPT_SUPPORT
  if (ompt_enabled.enabled) {
#if OMPT_OPTIONAL
    ompt_data_t *my_task_data;
    ompt_data_t *my_parallel_data;
    void *codeptr = NULL;
    int ds_tid = this_thr->th.th_info.ds.ds_tid;
    if (KMP_MASTER_TID(ds_tid) &&
        (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
         ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
      codeptr = team->t.ompt_team_info.master_return_address;
    my_task_data = OMPT_CUR_TASK_DATA(this_thr);
    my_parallel_data = OMPT_CUR_TEAM_DATA(this_thr);
    if (ompt_enabled.ompt_callback_sync_region) {
      ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
          ompt_sync_region_barrier_implicit, ompt_scope_begin, my_parallel_data,
          my_task_data, codeptr);
    }
    if (ompt_enabled.ompt_callback_sync_region_wait) {
      ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
          ompt_sync_region_barrier_implicit, ompt_scope_begin, my_parallel_data,
          my_task_data, codeptr);
    }
    if (!KMP_MASTER_TID(ds_tid))
      this_thr->th.ompt_thread_info.task_data = *OMPT_CUR_TASK_DATA(this_thr);
#endif
    this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier_implicit;
  }
#endif

  if (__kmp_tasking_mode == tskm_extra_barrier) {
    __kmp_tasking_barrier(team, this_thr, gtid);
    KA_TRACE(10, ("__kmp_join_barrier: T#%d(%d:%d) past tasking barrier\n",
                  gtid, team_id, tid));
  }
#ifdef KMP_DEBUG
  if (__kmp_tasking_mode != tskm_immediate_exec) {
    KA_TRACE(20, ("__kmp_join_barrier: T#%d, old team = %d, old task_team = "
                  "%p, th_task_team = %p\n",
                  __kmp_gtid_from_thread(this_thr), team_id,
                  team->t.t_task_team[this_thr->th.th_task_state],
                  this_thr->th.th_task_team));
    if (this_thr->th.th_task_team)
      KMP_DEBUG_ASSERT(this_thr->th.th_task_team ==
                       team->t.t_task_team[this_thr->th.th_task_state]);
  }
#endif /* KMP_DEBUG */

  /* Copy the blocktime info to the thread, where __kmp_wait_template() can
     access it when the team struct is not guaranteed to exist. Doing these
     loads causes a cache miss slows down EPCC parallel by 2x. As a workaround,
     we do not perform the copy if blocktime=infinite, since the values are not
     used by __kmp_wait_template() in that case. */
  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
#if KMP_USE_MONITOR
    this_thr->th.th_team_bt_intervals =
        team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
    this_thr->th.th_team_bt_set =
        team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
#else
    this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid);
#endif
  }

#if USE_ITT_BUILD
  if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
    __kmp_itt_barrier_starting(gtid, itt_sync_obj);
#endif /* USE_ITT_BUILD */

  switch (__kmp_barrier_gather_pattern[bs_forkjoin_barrier]) {
  case bp_dist_bar: {
    __kmp_dist_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid,
                              NULL USE_ITT_BUILD_ARG(itt_sync_obj));
    break;
  }
  case bp_hyper_bar: {
    KMP_ASSERT(__kmp_barrier_gather_branch_bits[bs_forkjoin_barrier]);
    __kmp_hyper_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid,
                               NULL USE_ITT_BUILD_ARG(itt_sync_obj));
    break;
  }
  case bp_hierarchical_bar: {
    __kmp_hierarchical_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid,
                                      NULL USE_ITT_BUILD_ARG(itt_sync_obj));
    break;
  }
  case bp_tree_bar: {
    KMP_ASSERT(__kmp_barrier_gather_branch_bits[bs_forkjoin_barrier]);
    __kmp_tree_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid,
                              NULL USE_ITT_BUILD_ARG(itt_sync_obj));
    break;
  }
  default: {
    __kmp_linear_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid,
                                NULL USE_ITT_BUILD_ARG(itt_sync_obj));
  }
  }

  /* From this point on, the team data structure may be deallocated at any time
     by the primary thread - it is unsafe to reference it in any of the worker
     threads. Any per-team data items that need to be referenced before the
     end of the barrier should be moved to the kmp_task_team_t structs.  */
  if (KMP_MASTER_TID(tid)) {
    if (__kmp_tasking_mode != tskm_immediate_exec) {
      __kmp_task_team_wait(this_thr, team USE_ITT_BUILD_ARG(itt_sync_obj));
    }
    if (__kmp_display_affinity) {
      KMP_CHECK_UPDATE(team->t.t_display_affinity, 0);
    }
#if KMP_STATS_ENABLED
    // Have primary thread flag the workers to indicate they are now waiting for
    // next parallel region, Also wake them up so they switch their timers to
    // idle.
    for (int i = 0; i < team->t.t_nproc; ++i) {
      kmp_info_t *team_thread = team->t.t_threads[i];
      if (team_thread == this_thr)
        continue;
      team_thread->th.th_stats->setIdleFlag();
      if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME &&
          team_thread->th.th_sleep_loc != NULL)
        __kmp_null_resume_wrapper(team_thread);
    }
#endif
#if USE_ITT_BUILD
    if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
      __kmp_itt_barrier_middle(gtid, itt_sync_obj);
#endif /* USE_ITT_BUILD */

#if USE_ITT_BUILD && USE_ITT_NOTIFY
    // Join barrier - report frame end
    if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
        __kmp_forkjoin_frames_mode &&
        (this_thr->th.th_teams_microtask == NULL || // either not in teams
         this_thr->th.th_teams_size.nteams == 1) && // or inside single team
        team->t.t_active_level == 1) {
      kmp_uint64 cur_time = __itt_get_timestamp();
      ident_t *loc = team->t.t_ident;
      kmp_info_t **other_threads = team->t.t_threads;
      int nproc = this_thr->th.th_team_nproc;
      int i;
      switch (__kmp_forkjoin_frames_mode) {
      case 1:
        __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0,
                               loc, nproc);
        break;
      case 2:
        __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1,
                               loc, nproc);
        break;
      case 3:
        if (__itt_metadata_add_ptr) {
          // Initialize with primary thread's wait time
          kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
          // Set arrive time to zero to be able to check it in
          // __kmp_invoke_task(); the same is done inside the loop below
          this_thr->th.th_bar_arrive_time = 0;
          for (i = 1; i < nproc; ++i) {
            delta += (cur_time - other_threads[i]->th.th_bar_arrive_time);
            other_threads[i]->th.th_bar_arrive_time = 0;
          }
          __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time,
                                       cur_time, delta, 0);
        }
        __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0,
                               loc, nproc);
        this_thr->th.th_frame_time = cur_time;
        break;
      }
    }
#endif /* USE_ITT_BUILD */
  }
#if USE_ITT_BUILD
  else {
    if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
      __kmp_itt_barrier_middle(gtid, itt_sync_obj);
  }
#endif /* USE_ITT_BUILD */

#if KMP_DEBUG
  if (KMP_MASTER_TID(tid)) {
    KA_TRACE(
        15,
        ("__kmp_join_barrier: T#%d(%d:%d) says all %d team threads arrived\n",
         gtid, team_id, tid, nproc));
  }
#endif /* KMP_DEBUG */

  // TODO now, mark worker threads as done so they may be disbanded
  KMP_MB(); // Flush all pending memory write invalidates.
  KA_TRACE(10,
           ("__kmp_join_barrier: T#%d(%d:%d) leaving\n", gtid, team_id, tid));

  ANNOTATE_BARRIER_END(&team->t.t_bar);
}

// TODO release worker threads' fork barriers as we are ready instead of all at
// once
void __kmp_fork_barrier(int gtid, int tid) {
  KMP_TIME_PARTITIONED_BLOCK(OMP_fork_barrier);
  KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER);
  kmp_info_t *this_thr = __kmp_threads[gtid];
  kmp_team_t *team = (tid == 0) ? this_thr->th.th_team : NULL;
#if USE_ITT_BUILD
  void *itt_sync_obj = NULL;
#endif /* USE_ITT_BUILD */
  if (team)
    ANNOTATE_BARRIER_END(&team->t.t_bar);

  KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) has arrived\n", gtid,
                (team != NULL) ? team->t.t_id : -1, tid));

  // th_team pointer only valid for primary thread here
  if (KMP_MASTER_TID(tid)) {
#if USE_ITT_BUILD && USE_ITT_NOTIFY
    if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
      // Create itt barrier object
      itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 1);
      __kmp_itt_barrier_middle(gtid, itt_sync_obj); // Call acquired/releasing
    }
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */

#ifdef KMP_DEBUG
    KMP_DEBUG_ASSERT(team);
    kmp_info_t **other_threads = team->t.t_threads;
    int i;

    // Verify state
    KMP_MB();

    for (i = 1; i < team->t.t_nproc; ++i) {
      KA_TRACE(500,
               ("__kmp_fork_barrier: T#%d(%d:0) checking T#%d(%d:%d) fork go "
                "== %u.\n",
                gtid, team->t.t_id, other_threads[i]->th.th_info.ds.ds_gtid,
                team->t.t_id, other_threads[i]->th.th_info.ds.ds_tid,
                other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go));
      KMP_DEBUG_ASSERT(
          (TCR_4(other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go) &
           ~(KMP_BARRIER_SLEEP_STATE)) == KMP_INIT_BARRIER_STATE);
      KMP_DEBUG_ASSERT(other_threads[i]->th.th_team == team);
    }
#endif

    if (__kmp_tasking_mode != tskm_immediate_exec) {
      // 0 indicates setup current task team if nthreads > 1
      __kmp_task_team_setup(this_thr, team, 0);
    }

    /* The primary thread may have changed its blocktime between join barrier
       and fork barrier. Copy the blocktime info to the thread, where
       __kmp_wait_template() can access it when the team struct is not
       guaranteed to exist. */
    // See note about the corresponding code in __kmp_join_barrier() being
    // performance-critical
    if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
#if KMP_USE_MONITOR
      this_thr->th.th_team_bt_intervals =
          team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
      this_thr->th.th_team_bt_set =
          team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
#else
      this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid);
#endif
    }
  } // primary thread

  switch (__kmp_barrier_release_pattern[bs_forkjoin_barrier]) {
  case bp_dist_bar: {
    __kmp_dist_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid,
                               TRUE USE_ITT_BUILD_ARG(NULL));
    break;
  }
  case bp_hyper_bar: {
    KMP_ASSERT(__kmp_barrier_release_branch_bits[bs_forkjoin_barrier]);
    __kmp_hyper_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid,
                                TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
    break;
  }
  case bp_hierarchical_bar: {
    __kmp_hierarchical_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid,
                                       TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
    break;
  }
  case bp_tree_bar: {
    KMP_ASSERT(__kmp_barrier_release_branch_bits[bs_forkjoin_barrier]);
    __kmp_tree_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid,
                               TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
    break;
  }
  default: {
    __kmp_linear_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid,
                                 TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
  }
  }

#if OMPT_SUPPORT
  if (ompt_enabled.enabled &&
      this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) {
    int ds_tid = this_thr->th.th_info.ds.ds_tid;
    ompt_data_t *task_data = (team)
                                 ? OMPT_CUR_TASK_DATA(this_thr)
                                 : &(this_thr->th.ompt_thread_info.task_data);
    this_thr->th.ompt_thread_info.state = ompt_state_overhead;
#if OMPT_OPTIONAL
    void *codeptr = NULL;
    if (KMP_MASTER_TID(ds_tid) &&
        (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
         ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
      codeptr = team ? team->t.ompt_team_info.master_return_address : NULL;
    if (ompt_enabled.ompt_callback_sync_region_wait) {
      ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
          ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
          codeptr);
    }
    if (ompt_enabled.ompt_callback_sync_region) {
      ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
          ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
          codeptr);
    }
#endif
    if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) {
      ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
          ompt_scope_end, NULL, task_data, 0, ds_tid,
          ompt_task_implicit); // TODO: Can this be ompt_task_initial?
    }
  }
#endif

  // Early exit for reaping threads releasing forkjoin barrier
  if (TCR_4(__kmp_global.g.g_done)) {
    this_thr->th.th_task_team = NULL;

#if USE_ITT_BUILD && USE_ITT_NOTIFY
    if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
      if (!KMP_MASTER_TID(tid)) {
        itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
        if (itt_sync_obj)
          __kmp_itt_barrier_finished(gtid, itt_sync_obj);
      }
    }
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
    KA_TRACE(10, ("__kmp_fork_barrier: T#%d is leaving early\n", gtid));
    return;
  }

  /* We can now assume that a valid team structure has been allocated by the
     primary thread and propagated to all worker threads. The current thread,
     however, may not be part of the team, so we can't blindly assume that the
     team pointer is non-null.  */
  team = (kmp_team_t *)TCR_PTR(this_thr->th.th_team);
  KMP_DEBUG_ASSERT(team != NULL);
  tid = __kmp_tid_from_gtid(gtid);

#if KMP_BARRIER_ICV_PULL
  /* Primary thread's copy of the ICVs was set up on the implicit taskdata in
     __kmp_reinitialize_team. __kmp_fork_call() assumes the primary thread's
     implicit task has this data before this function is called. We cannot
     modify __kmp_fork_call() to look at the fixed ICVs in the primary thread's
     thread struct, because it is not always the case that the threads arrays
     have been allocated when __kmp_fork_call() is executed. */
  {
    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
    if (!KMP_MASTER_TID(tid)) { // primary thread already has ICVs
      // Copy the initial ICVs from the primary thread's thread struct to the
      // implicit task for this tid.
      KA_TRACE(10,
               ("__kmp_fork_barrier: T#%d(%d) is PULLing ICVs\n", gtid, tid));
      __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team,
                               tid, FALSE);
      copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
                &team->t.t_threads[0]
                     ->th.th_bar[bs_forkjoin_barrier]
                     .bb.th_fixed_icvs);
    }
  }
#endif // KMP_BARRIER_ICV_PULL

  if (__kmp_tasking_mode != tskm_immediate_exec) {
    __kmp_task_team_sync(this_thr, team);
  }

#if KMP_AFFINITY_SUPPORTED
  kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
  if (proc_bind == proc_bind_intel) {
    // Call dynamic affinity settings
    if (__kmp_affinity_type == affinity_balanced && team->t.t_size_changed) {
      __kmp_balanced_affinity(this_thr, team->t.t_nproc);
    }
  } else if (proc_bind != proc_bind_false) {
    if (this_thr->th.th_new_place == this_thr->th.th_current_place) {
      KA_TRACE(100, ("__kmp_fork_barrier: T#%d already in correct place %d\n",
                     __kmp_gtid_from_thread(this_thr),
                     this_thr->th.th_current_place));
    } else {
      __kmp_affinity_set_place(gtid);
    }
  }
#endif // KMP_AFFINITY_SUPPORTED
  // Perform the display affinity functionality
  if (__kmp_display_affinity) {
    if (team->t.t_display_affinity
#if KMP_AFFINITY_SUPPORTED
        || (__kmp_affinity_type == affinity_balanced && team->t.t_size_changed)
#endif
    ) {
      // NULL means use the affinity-format-var ICV
      __kmp_aux_display_affinity(gtid, NULL);
      this_thr->th.th_prev_num_threads = team->t.t_nproc;
      this_thr->th.th_prev_level = team->t.t_level;
    }
  }
  if (!KMP_MASTER_TID(tid))
    KMP_CHECK_UPDATE(this_thr->th.th_def_allocator, team->t.t_def_allocator);

#if USE_ITT_BUILD && USE_ITT_NOTIFY
  if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
    if (!KMP_MASTER_TID(tid)) {
      // Get correct barrier object
      itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
      __kmp_itt_barrier_finished(gtid, itt_sync_obj); // Workers call acquired
    } // (prepare called inside barrier_release)
  }
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
  ANNOTATE_BARRIER_END(&team->t.t_bar);
  KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) is leaving\n", gtid,
                team->t.t_id, tid));
}

void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
                          kmp_internal_control_t *new_icvs, ident_t *loc) {
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_setup_icv_copy);

  KMP_DEBUG_ASSERT(team && new_nproc && new_icvs);
  KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);

/* Primary thread's copy of the ICVs was set up on the implicit taskdata in
   __kmp_reinitialize_team. __kmp_fork_call() assumes the primary thread's
   implicit task has this data before this function is called. */
#if KMP_BARRIER_ICV_PULL
  /* Copy ICVs to primary thread's thread structure into th_fixed_icvs (which
     remains untouched), where all of the worker threads can access them and
     make their own copies after the barrier. */
  KMP_DEBUG_ASSERT(team->t.t_threads[0]); // The threads arrays should be
  // allocated at this point
  copy_icvs(
      &team->t.t_threads[0]->th.th_bar[bs_forkjoin_barrier].bb.th_fixed_icvs,
      new_icvs);
  KF_TRACE(10, ("__kmp_setup_icv_copy: PULL: T#%d this_thread=%p team=%p\n", 0,
                team->t.t_threads[0], team));
#elif KMP_BARRIER_ICV_PUSH
  // The ICVs will be propagated in the fork barrier, so nothing needs to be
  // done here.
  KF_TRACE(10, ("__kmp_setup_icv_copy: PUSH: T#%d this_thread=%p team=%p\n", 0,
                team->t.t_threads[0], team));
#else
  // Copy the ICVs to each of the non-primary threads.  This takes O(nthreads)
  // time.
  ngo_load(new_icvs);
  KMP_DEBUG_ASSERT(team->t.t_threads[0]); // The threads arrays should be
  // allocated at this point
  for (int f = 1; f < new_nproc; ++f) { // Skip the primary thread
    // TODO: GEH - pass in better source location info since usually NULL here
    KF_TRACE(10, ("__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n",
                  f, team->t.t_threads[f], team));
    __kmp_init_implicit_task(loc, team->t.t_threads[f], team, f, FALSE);
    ngo_store_icvs(&team->t.t_implicit_task_taskdata[f].td_icvs, new_icvs);
    KF_TRACE(10, ("__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n",
                  f, team->t.t_threads[f], team));
  }
  ngo_sync();
#endif // KMP_BARRIER_ICV_PULL
}