54 void task_group_context_impl::destroy(d1::task_group_context& ctx) {  in destroy()  argument
55     __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);  in destroy()
57     if (ctx.my_context_list != nullptr) {  in destroy()
58 …__TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) == d1::task_group_context::state::bound,…  in destroy()
60         ctx.my_context_list->remove(ctx.my_node);  in destroy()
62     d1::cpu_ctl_env* ctl = reinterpret_cast<d1::cpu_ctl_env*>(&ctx.my_cpu_ctl_env);  in destroy()
68     auto exception = ctx.my_exception.load(std::memory_order_relaxed);  in destroy()
72     ITT_STACK_DESTROY(ctx.my_itt_caller);  in destroy()
74     poison_pointer(ctx.my_parent);  in destroy()
75     poison_pointer(ctx.my_context_list);  in destroy()
76     poison_pointer(ctx.my_node.my_next_node);  in destroy()
77     poison_pointer(ctx.my_node.my_prev_node);  in destroy()
78     poison_pointer(ctx.my_exception);  in destroy()
79     poison_pointer(ctx.my_itt_caller);  in destroy()
81     ctx.my_state.store(d1::task_group_context::state::dead, std::memory_order_release);  in destroy()
84 void task_group_context_impl::initialize(d1::task_group_context& ctx) {  in initialize()  argument
85     ITT_TASK_GROUP(&ctx, ctx.my_name, nullptr);  in initialize()
87     ctx.my_node.my_next_node = &ctx.my_node;  in initialize()
88     ctx.my_node.my_prev_node = &ctx.my_node;  in initialize()
89     ctx.my_cpu_ctl_env = 0;  in initialize()
90     ctx.my_cancellation_requested = 0;  in initialize()
91     ctx.my_may_have_children.store(0, std::memory_order_relaxed);  in initialize()
93     ctx.my_state.store(d1::task_group_context::state::created, std::memory_order_relaxed);  in initialize()
94     ctx.my_parent = nullptr;  in initialize()
95     ctx.my_context_list = nullptr;  in initialize()
96     ctx.my_exception.store(nullptr, std::memory_order_relaxed);  in initialize()
97     ctx.my_itt_caller = nullptr;  in initialize()
99 …static_assert(sizeof(d1::cpu_ctl_env) <= sizeof(ctx.my_cpu_ctl_env), "FPU settings storage does no…  in initialize()
100     d1::cpu_ctl_env* ctl = new (&ctx.my_cpu_ctl_env) d1::cpu_ctl_env;  in initialize()
101     if (ctx.my_traits.fp_settings)  in initialize()
105 void task_group_context_impl::register_with(d1::task_group_context& ctx, thread_data* td) {  in register_with()  argument
106     __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);  in register_with()
108     ctx.my_context_list = td->my_context_list;  in register_with()
110     ctx.my_context_list->push_front(ctx.my_node);  in register_with()
113 void task_group_context_impl::bind_to_impl(d1::task_group_context& ctx, thread_data* td) {  in bind_to_impl()  argument
114     __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);  in bind_to_impl()
115 …__TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) == d1::task_group_context::state::locked…  in bind_to_impl()
116     __TBB_ASSERT(!ctx.my_parent, "Parent is set before initial binding");  in bind_to_impl()
118     ctx.my_parent = td->my_task_dispatcher->m_execute_data_ext.context;  in bind_to_impl()
119     __TBB_ASSERT(ctx.my_parent, nullptr);  in bind_to_impl()
122     if (!ctx.my_traits.fp_settings)  in bind_to_impl()
123         copy_fp_settings(ctx, *ctx.my_parent);  in bind_to_impl()
126 …if (ctx.my_parent->my_may_have_children.load(std::memory_order_relaxed) != d1::task_group_context:…  in bind_to_impl()
127 …ctx.my_parent->my_may_have_children.store(d1::task_group_context::may_have_children, std::memory_o…  in bind_to_impl()
129     if (ctx.my_parent->my_parent) {  in bind_to_impl()
141 …uintptr_t local_count_snapshot = ctx.my_parent->my_context_list->epoch.load(std::memory_order_acqu…  in bind_to_impl()
144 …ctx.my_cancellation_requested.store(ctx.my_parent->my_cancellation_requested.load(std::memory_orde…  in bind_to_impl()
145         register_with(ctx, td); // Issues full fence  in bind_to_impl()
154 …ctx.my_cancellation_requested.store(ctx.my_parent->my_cancellation_requested.load(std::memory_orde…  in bind_to_impl()
157         register_with(ctx, td); // Issues full fence  in bind_to_impl()
161 …ctx.my_cancellation_requested.store(ctx.my_parent->my_cancellation_requested.load(std::memory_orde…  in bind_to_impl()
165 void task_group_context_impl::bind_to(d1::task_group_context& ctx, thread_data* td) {  in bind_to()  argument
166     d1::task_group_context::state state = ctx.my_state.load(std::memory_order_acquire);  in bind_to()
170 …((std::atomic<typename std::underlying_type<d1::task_group_context::state>::type>&)ctx.my_state).c…  in bind_to()
174             ctx.my_state.compare_exchange_strong(state, d1::task_group_context::state::locked)  in bind_to()
182 …k_dispatcher->m_execute_data_ext.context == td->my_arena->my_default_ctx || !ctx.my_traits.bound) {  in bind_to()
183                 if (!ctx.my_traits.fp_settings) {  in bind_to()
184                     copy_fp_settings(ctx, *td->my_arena->my_default_ctx);  in bind_to()
188                 bind_to_impl(ctx, td);  in bind_to()
191             ITT_STACK_CREATE(ctx.my_itt_caller);  in bind_to()
192             ctx.my_state.store(release_state, std::memory_order_release);  in bind_to()
194         spin_wait_while_eq(ctx.my_state, d1::task_group_context::state::locked);  in bind_to()
196 …__TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) != d1::task_group_context::state::create…  in bind_to()
197 …__TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) != d1::task_group_context::state::locked…  in bind_to()
200 void task_group_context_impl::propagate_task_group_state(d1::task_group_context& ctx, std::atomic<s…  in propagate_task_group_state()  argument
201     __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);  in propagate_task_group_state()
215     if ((ctx.*mptr_state).load(std::memory_order_relaxed) != new_state && &ctx != &src) {  in propagate_task_group_state()
216 …for (d1::task_group_context* ancestor = ctx.my_parent; ancestor != nullptr; ancestor = ancestor->m…  in propagate_task_group_state()
218                 for (d1::task_group_context* c = &ctx; c != ancestor; c = c->my_parent)  in propagate_task_group_state()
226 bool task_group_context_impl::cancel_group_execution(d1::task_group_context& ctx) {  in cancel_group_execution()  argument
227     __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);  in cancel_group_execution()
228 …__TBB_ASSERT(ctx.my_cancellation_requested.load(std::memory_order_relaxed) <= 1, "The cancellation…  in cancel_group_execution()
229 …if (ctx.my_cancellation_requested.load(std::memory_order_relaxed) || ctx.my_cancellation_requested…  in cancel_group_execution()
235 …->propagate_task_group_state(&d1::task_group_context::my_cancellation_requested, ctx, uint32_t(1));  in cancel_group_execution()
239 bool task_group_context_impl::is_group_execution_cancelled(const d1::task_group_context& ctx) {  in is_group_execution_cancelled()  argument
240     return ctx.my_cancellation_requested.load(std::memory_order_relaxed) != 0;  in is_group_execution_cancelled()
244 void task_group_context_impl::reset(d1::task_group_context& ctx) {  in reset()  argument
245     __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);  in reset()
250     auto exception = ctx.my_exception.load(std::memory_order_relaxed);  in reset()
253         ctx.my_exception.store(nullptr, std::memory_order_relaxed);  in reset()
255     ctx.my_cancellation_requested = 0;  in reset()
259 void task_group_context_impl::capture_fp_settings(d1::task_group_context& ctx) {  in capture_fp_settings()  argument
260     __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);  in capture_fp_settings()
264     d1::cpu_ctl_env* ctl = reinterpret_cast<d1::cpu_ctl_env*>(&ctx.my_cpu_ctl_env);  in capture_fp_settings()
265     if (!ctx.my_traits.fp_settings) {  in capture_fp_settings()
266         ctl = new (&ctx.my_cpu_ctl_env) d1::cpu_ctl_env;  in capture_fp_settings()
267         ctx.my_traits.fp_settings = true;  in capture_fp_settings()
272 void task_group_context_impl::copy_fp_settings(d1::task_group_context& ctx, const d1::task_group_co…  in copy_fp_settings()  argument
273     __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);  in copy_fp_settings()
274     __TBB_ASSERT(!ctx.my_traits.fp_settings, "The context already has FPU settings.");  in copy_fp_settings()
278     new (&ctx.my_cpu_ctl_env) d1::cpu_ctl_env(*src_ctl);  in copy_fp_settings()
279     ctx.my_traits.fp_settings = true;  in copy_fp_settings()
336 void __TBB_EXPORTED_FUNC initialize(d1::task_group_context& ctx) {  in initialize()  argument
337     task_group_context_impl::initialize(ctx);  in initialize()
339 void __TBB_EXPORTED_FUNC destroy(d1::task_group_context& ctx) {  in destroy()  argument
340     task_group_context_impl::destroy(ctx);  in destroy()
342 void __TBB_EXPORTED_FUNC reset(d1::task_group_context& ctx) {  in reset()  argument
343     task_group_context_impl::reset(ctx);  in reset()
345 bool __TBB_EXPORTED_FUNC cancel_group_execution(d1::task_group_context& ctx) {  in cancel_group_execution()  argument
346     return task_group_context_impl::cancel_group_execution(ctx);  in cancel_group_execution()
348 bool __TBB_EXPORTED_FUNC is_group_execution_cancelled(d1::task_group_context& ctx) {  in is_group_execution_cancelled()  argument
349     return task_group_context_impl::is_group_execution_cancelled(ctx);  in is_group_execution_cancelled()
351 void __TBB_EXPORTED_FUNC capture_fp_settings(d1::task_group_context& ctx) {  in capture_fp_settings()  argument
352     task_group_context_impl::capture_fp_settings(ctx);  in capture_fp_settings()