187c2fdc1SPavel Kumbrasev /*
287c2fdc1SPavel Kumbrasev     Copyright (c) 2023 Intel Corporation
387c2fdc1SPavel Kumbrasev 
487c2fdc1SPavel Kumbrasev     Licensed under the Apache License, Version 2.0 (the "License");
587c2fdc1SPavel Kumbrasev     you may not use this file except in compliance with the License.
687c2fdc1SPavel Kumbrasev     You may obtain a copy of the License at
787c2fdc1SPavel Kumbrasev 
887c2fdc1SPavel Kumbrasev         http://www.apache.org/licenses/LICENSE-2.0
987c2fdc1SPavel Kumbrasev 
1087c2fdc1SPavel Kumbrasev     Unless required by applicable law or agreed to in writing, software
1187c2fdc1SPavel Kumbrasev     distributed under the License is distributed on an "AS IS" BASIS,
1287c2fdc1SPavel Kumbrasev     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1387c2fdc1SPavel Kumbrasev     See the License for the specific language governing permissions and
1487c2fdc1SPavel Kumbrasev     limitations under the License.
1587c2fdc1SPavel Kumbrasev */
1687c2fdc1SPavel Kumbrasev 
1787c2fdc1SPavel Kumbrasev #ifndef __TBB_task_emulation_layer_H
1887c2fdc1SPavel Kumbrasev #define __TBB_task_emulation_layer_H
1987c2fdc1SPavel Kumbrasev 
2087c2fdc1SPavel Kumbrasev #include "tbb/task_group.h"
2187c2fdc1SPavel Kumbrasev #include "tbb/task_arena.h"
2287c2fdc1SPavel Kumbrasev 
2387c2fdc1SPavel Kumbrasev #include <atomic>
2487c2fdc1SPavel Kumbrasev 
2587c2fdc1SPavel Kumbrasev namespace task_emulation {
2687c2fdc1SPavel Kumbrasev 
2787c2fdc1SPavel Kumbrasev struct task_group_pool {
task_group_pooltask_group_pool2887c2fdc1SPavel Kumbrasev     task_group_pool() : pool_size(std::thread::hardware_concurrency()), task_submitters(new tbb::task_group[pool_size]) {}
2987c2fdc1SPavel Kumbrasev 
~task_group_pooltask_group_pool3087c2fdc1SPavel Kumbrasev     ~task_group_pool() {
3187c2fdc1SPavel Kumbrasev         for (std::size_t i = 0; i < pool_size; ++i) {
3287c2fdc1SPavel Kumbrasev             task_submitters[i].wait();
3387c2fdc1SPavel Kumbrasev         }
3487c2fdc1SPavel Kumbrasev 
3587c2fdc1SPavel Kumbrasev         delete [] task_submitters;
3687c2fdc1SPavel Kumbrasev     }
3787c2fdc1SPavel Kumbrasev 
3887c2fdc1SPavel Kumbrasev     tbb::task_group& operator[] (std::size_t idx) { return task_submitters[idx]; }
3987c2fdc1SPavel Kumbrasev 
4087c2fdc1SPavel Kumbrasev     const std::size_t pool_size;
4187c2fdc1SPavel Kumbrasev     tbb::task_group* task_submitters;
4287c2fdc1SPavel Kumbrasev };
4387c2fdc1SPavel Kumbrasev 
4487c2fdc1SPavel Kumbrasev static task_group_pool tg_pool;
4587c2fdc1SPavel Kumbrasev 
4687c2fdc1SPavel Kumbrasev class base_task {
4787c2fdc1SPavel Kumbrasev public:
4887c2fdc1SPavel Kumbrasev     base_task() = default;
4987c2fdc1SPavel Kumbrasev 
base_task(const base_task & t)50*91ba7e7eSPavel Kumbrasev     base_task(const base_task& t) : m_type(t.m_type), m_parent(t.m_parent), m_child_counter(t.m_child_counter.load())
5187c2fdc1SPavel Kumbrasev     {}
5287c2fdc1SPavel Kumbrasev 
5387c2fdc1SPavel Kumbrasev     virtual ~base_task() = default;
5487c2fdc1SPavel Kumbrasev 
operator()5587c2fdc1SPavel Kumbrasev     void operator() () const {
56*91ba7e7eSPavel Kumbrasev         task_type type_snapshot = m_type;
57*91ba7e7eSPavel Kumbrasev 
58*91ba7e7eSPavel Kumbrasev         base_task* bypass = const_cast<base_task*>(this)->execute();
59*91ba7e7eSPavel Kumbrasev 
60*91ba7e7eSPavel Kumbrasev         if (m_parent && m_type != task_type::recycled) {
61*91ba7e7eSPavel Kumbrasev             if (m_parent->remove_child_reference() == 0) {
62d489c1f7SPavel Kumbrasev                 m_parent->operator()();
63d489c1f7SPavel Kumbrasev             }
64d489c1f7SPavel Kumbrasev         }
65d489c1f7SPavel Kumbrasev 
66*91ba7e7eSPavel Kumbrasev         if (m_type == task_type::allocated) {
67d489c1f7SPavel Kumbrasev             delete this;
6887c2fdc1SPavel Kumbrasev         }
69*91ba7e7eSPavel Kumbrasev 
70*91ba7e7eSPavel Kumbrasev         if (bypass != nullptr) {
71*91ba7e7eSPavel Kumbrasev             m_type = type_snapshot;
72*91ba7e7eSPavel Kumbrasev 
73*91ba7e7eSPavel Kumbrasev             // Bypass is not supported by task_emulation and next_task executed directly.
74*91ba7e7eSPavel Kumbrasev             // However, the old-TBB bypass behavior can be achieved with
75*91ba7e7eSPavel Kumbrasev             // `return task_group::defer()` (check Migration Guide).
76*91ba7e7eSPavel Kumbrasev             // Consider submit another task if recursion call is not acceptable
77*91ba7e7eSPavel Kumbrasev             // i.e. instead of Direct Body call
78*91ba7e7eSPavel Kumbrasev             // submit task_emulation::run_task();
79*91ba7e7eSPavel Kumbrasev             bypass->operator()();
80*91ba7e7eSPavel Kumbrasev         }
8187c2fdc1SPavel Kumbrasev     }
8287c2fdc1SPavel Kumbrasev 
83*91ba7e7eSPavel Kumbrasev     virtual base_task* execute() = 0;
8487c2fdc1SPavel Kumbrasev 
8587c2fdc1SPavel Kumbrasev     template <typename C, typename... Args>
allocate_continuation(std::uint64_t ref,Args &&...args)86d489c1f7SPavel Kumbrasev     C* allocate_continuation(std::uint64_t ref, Args&&... args) {
8787c2fdc1SPavel Kumbrasev         C* continuation = new C{std::forward<Args>(args)...};
88*91ba7e7eSPavel Kumbrasev         continuation->m_type = task_type::allocated;
8987c2fdc1SPavel Kumbrasev         continuation->reset_parent(reset_parent());
9087c2fdc1SPavel Kumbrasev         continuation->m_child_counter = ref;
9187c2fdc1SPavel Kumbrasev         return continuation;
9287c2fdc1SPavel Kumbrasev     }
9387c2fdc1SPavel Kumbrasev 
9487c2fdc1SPavel Kumbrasev     template <typename F, typename... Args>
create_child(Args &&...args)95d489c1f7SPavel Kumbrasev     F create_child(Args&&... args) {
96d489c1f7SPavel Kumbrasev         return create_child_impl<F>(std::forward<Args>(args)...);
97d489c1f7SPavel Kumbrasev     }
98d489c1f7SPavel Kumbrasev 
99d489c1f7SPavel Kumbrasev     template <typename F, typename... Args>
create_child_and_increment(Args &&...args)100d489c1f7SPavel Kumbrasev     F create_child_and_increment(Args&&... args) {
101*91ba7e7eSPavel Kumbrasev         add_child_reference();
102d489c1f7SPavel Kumbrasev         return create_child_impl<F>(std::forward<Args>(args)...);
103d489c1f7SPavel Kumbrasev     }
104d489c1f7SPavel Kumbrasev 
105d489c1f7SPavel Kumbrasev     template <typename F, typename... Args>
allocate_child(Args &&...args)106d489c1f7SPavel Kumbrasev     F* allocate_child(Args&&... args) {
107d489c1f7SPavel Kumbrasev         return allocate_child_impl<F>(std::forward<Args>(args)...);
108d489c1f7SPavel Kumbrasev     }
109d489c1f7SPavel Kumbrasev 
110d489c1f7SPavel Kumbrasev     template <typename F, typename... Args>
allocate_child_and_increment(Args &&...args)111d489c1f7SPavel Kumbrasev     F* allocate_child_and_increment(Args&&... args) {
112*91ba7e7eSPavel Kumbrasev         add_child_reference();
113d489c1f7SPavel Kumbrasev         return allocate_child_impl<F>(std::forward<Args>(args)...);
11487c2fdc1SPavel Kumbrasev     }
11587c2fdc1SPavel Kumbrasev 
11687c2fdc1SPavel Kumbrasev     template <typename C>
recycle_as_child_of(C & c)117d489c1f7SPavel Kumbrasev     void recycle_as_child_of(C& c) {
118*91ba7e7eSPavel Kumbrasev         m_type = task_type::recycled;
11987c2fdc1SPavel Kumbrasev         reset_parent(&c);
12087c2fdc1SPavel Kumbrasev     }
12187c2fdc1SPavel Kumbrasev 
recycle_as_continuation()122d489c1f7SPavel Kumbrasev     void recycle_as_continuation() {
123*91ba7e7eSPavel Kumbrasev         m_type = task_type::recycled;
124d489c1f7SPavel Kumbrasev     }
125d489c1f7SPavel Kumbrasev 
add_child_reference()126*91ba7e7eSPavel Kumbrasev     void add_child_reference() {
12787c2fdc1SPavel Kumbrasev         ++m_child_counter;
12887c2fdc1SPavel Kumbrasev     }
12987c2fdc1SPavel Kumbrasev 
remove_child_reference()130*91ba7e7eSPavel Kumbrasev     std::uint64_t remove_child_reference() {
131d489c1f7SPavel Kumbrasev         return --m_child_counter;
132d489c1f7SPavel Kumbrasev     }
133d489c1f7SPavel Kumbrasev 
134d489c1f7SPavel Kumbrasev protected:
135d489c1f7SPavel Kumbrasev     enum class task_type {
136*91ba7e7eSPavel Kumbrasev         stack_based,
137d489c1f7SPavel Kumbrasev         allocated,
138*91ba7e7eSPavel Kumbrasev         recycled
139d489c1f7SPavel Kumbrasev     };
140d489c1f7SPavel Kumbrasev 
141*91ba7e7eSPavel Kumbrasev     mutable task_type m_type;
142d489c1f7SPavel Kumbrasev 
14387c2fdc1SPavel Kumbrasev private:
14487c2fdc1SPavel Kumbrasev     template <typename F, typename... Args>
14587c2fdc1SPavel Kumbrasev     friend F create_root_task(tbb::task_group& tg, Args&&... args);
14687c2fdc1SPavel Kumbrasev 
147d489c1f7SPavel Kumbrasev     template <typename F, typename... Args>
148d489c1f7SPavel Kumbrasev     friend F* allocate_root_task(tbb::task_group& tg, Args&&... args);
14987c2fdc1SPavel Kumbrasev 
150d489c1f7SPavel Kumbrasev     template <typename F, typename... Args>
create_child_impl(Args &&...args)151d489c1f7SPavel Kumbrasev     F create_child_impl(Args&&... args) {
152d489c1f7SPavel Kumbrasev         F obj{std::forward<Args>(args)...};
153*91ba7e7eSPavel Kumbrasev         obj.m_type = task_type::stack_based;
154d489c1f7SPavel Kumbrasev         obj.reset_parent(this);
155d489c1f7SPavel Kumbrasev         return obj;
15687c2fdc1SPavel Kumbrasev     }
157d489c1f7SPavel Kumbrasev 
158d489c1f7SPavel Kumbrasev     template <typename F, typename... Args>
allocate_child_impl(Args &&...args)159d489c1f7SPavel Kumbrasev     F* allocate_child_impl(Args&&... args) {
160d489c1f7SPavel Kumbrasev         F* obj = new F{std::forward<Args>(args)...};
161d489c1f7SPavel Kumbrasev         obj->m_type = task_type::allocated;
162d489c1f7SPavel Kumbrasev         obj->reset_parent(this);
163d489c1f7SPavel Kumbrasev         return obj;
16487c2fdc1SPavel Kumbrasev     }
16587c2fdc1SPavel Kumbrasev 
16687c2fdc1SPavel Kumbrasev     base_task* reset_parent(base_task* ptr = nullptr) {
16787c2fdc1SPavel Kumbrasev         auto p = m_parent;
16887c2fdc1SPavel Kumbrasev         m_parent = ptr;
16987c2fdc1SPavel Kumbrasev         return p;
17087c2fdc1SPavel Kumbrasev     }
17187c2fdc1SPavel Kumbrasev 
17287c2fdc1SPavel Kumbrasev     base_task* m_parent{nullptr};
17387c2fdc1SPavel Kumbrasev     std::atomic<std::uint64_t> m_child_counter{0};
17487c2fdc1SPavel Kumbrasev };
17587c2fdc1SPavel Kumbrasev 
17687c2fdc1SPavel Kumbrasev class root_task : public base_task {
17787c2fdc1SPavel Kumbrasev public:
root_task(tbb::task_group & tg)17887c2fdc1SPavel Kumbrasev     root_task(tbb::task_group& tg) : m_tg(tg), m_callback(m_tg.defer([] { /* Create empty callback to preserve reference for wait. */})) {
179*91ba7e7eSPavel Kumbrasev         add_child_reference();
180*91ba7e7eSPavel Kumbrasev         m_type = base_task::task_type::allocated;
18187c2fdc1SPavel Kumbrasev     }
18287c2fdc1SPavel Kumbrasev 
18387c2fdc1SPavel Kumbrasev private:
execute()184*91ba7e7eSPavel Kumbrasev     base_task* execute() override {
18587c2fdc1SPavel Kumbrasev         m_tg.run(std::move(m_callback));
186*91ba7e7eSPavel Kumbrasev         return nullptr;
18787c2fdc1SPavel Kumbrasev     }
18887c2fdc1SPavel Kumbrasev 
18987c2fdc1SPavel Kumbrasev     tbb::task_group& m_tg;
19087c2fdc1SPavel Kumbrasev     tbb::task_handle m_callback;
19187c2fdc1SPavel Kumbrasev };
19287c2fdc1SPavel Kumbrasev 
19387c2fdc1SPavel Kumbrasev template <typename F, typename... Args>
create_root_task(tbb::task_group & tg,Args &&...args)19487c2fdc1SPavel Kumbrasev F create_root_task(tbb::task_group& tg, Args&&... args) {
19587c2fdc1SPavel Kumbrasev     F obj{std::forward<Args>(args)...};
196*91ba7e7eSPavel Kumbrasev     obj.m_type = base_task::task_type::stack_based;
19787c2fdc1SPavel Kumbrasev     obj.reset_parent(new root_task{tg});
19887c2fdc1SPavel Kumbrasev     return obj;
19987c2fdc1SPavel Kumbrasev }
20087c2fdc1SPavel Kumbrasev 
201d489c1f7SPavel Kumbrasev template <typename F, typename... Args>
allocate_root_task(tbb::task_group & tg,Args &&...args)202d489c1f7SPavel Kumbrasev F* allocate_root_task(tbb::task_group& tg, Args&&... args) {
203d489c1f7SPavel Kumbrasev     F* obj = new F{std::forward<Args>(args)...};
204d489c1f7SPavel Kumbrasev     obj->m_type = base_task::task_type::allocated;
205d489c1f7SPavel Kumbrasev     obj->reset_parent(new root_task{tg});
206d489c1f7SPavel Kumbrasev     return obj;
207d489c1f7SPavel Kumbrasev }
208d489c1f7SPavel Kumbrasev 
20987c2fdc1SPavel Kumbrasev template <typename F>
run_task(F && f)21087c2fdc1SPavel Kumbrasev void run_task(F&& f) {
21187c2fdc1SPavel Kumbrasev     tg_pool[tbb::this_task_arena::current_thread_index()].run(std::forward<F>(f));
21287c2fdc1SPavel Kumbrasev }
213d489c1f7SPavel Kumbrasev 
214d489c1f7SPavel Kumbrasev template <typename F>
run_task(F * f)215d489c1f7SPavel Kumbrasev void run_task(F* f) {
216d489c1f7SPavel Kumbrasev     tg_pool[tbb::this_task_arena::current_thread_index()].run(std::ref(*f));
217d489c1f7SPavel Kumbrasev }
218d489c1f7SPavel Kumbrasev 
219d489c1f7SPavel Kumbrasev template <typename F>
run_and_wait(tbb::task_group & tg,F * f)220d489c1f7SPavel Kumbrasev void run_and_wait(tbb::task_group& tg, F* f) {
221d489c1f7SPavel Kumbrasev    tg.run_and_wait(std::ref(*f));
222d489c1f7SPavel Kumbrasev }
22387c2fdc1SPavel Kumbrasev } // namespace task_emulation
22487c2fdc1SPavel Kumbrasev 
22587c2fdc1SPavel Kumbrasev #endif // __TBB_task_emulation_layer_H
226