187c2fdc1SPavel Kumbrasev /*
287c2fdc1SPavel Kumbrasev Copyright (c) 2023 Intel Corporation
387c2fdc1SPavel Kumbrasev
487c2fdc1SPavel Kumbrasev Licensed under the Apache License, Version 2.0 (the "License");
587c2fdc1SPavel Kumbrasev you may not use this file except in compliance with the License.
687c2fdc1SPavel Kumbrasev You may obtain a copy of the License at
787c2fdc1SPavel Kumbrasev
887c2fdc1SPavel Kumbrasev http://www.apache.org/licenses/LICENSE-2.0
987c2fdc1SPavel Kumbrasev
1087c2fdc1SPavel Kumbrasev Unless required by applicable law or agreed to in writing, software
1187c2fdc1SPavel Kumbrasev distributed under the License is distributed on an "AS IS" BASIS,
1287c2fdc1SPavel Kumbrasev WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1387c2fdc1SPavel Kumbrasev See the License for the specific language governing permissions and
1487c2fdc1SPavel Kumbrasev limitations under the License.
1587c2fdc1SPavel Kumbrasev */
1687c2fdc1SPavel Kumbrasev
1787c2fdc1SPavel Kumbrasev #ifndef __TBB_task_emulation_layer_H
1887c2fdc1SPavel Kumbrasev #define __TBB_task_emulation_layer_H
1987c2fdc1SPavel Kumbrasev
2087c2fdc1SPavel Kumbrasev #include "tbb/task_group.h"
2187c2fdc1SPavel Kumbrasev #include "tbb/task_arena.h"
2287c2fdc1SPavel Kumbrasev
2387c2fdc1SPavel Kumbrasev #include <atomic>
2487c2fdc1SPavel Kumbrasev
2587c2fdc1SPavel Kumbrasev namespace task_emulation {
2687c2fdc1SPavel Kumbrasev
2787c2fdc1SPavel Kumbrasev struct task_group_pool {
task_group_pooltask_group_pool2887c2fdc1SPavel Kumbrasev task_group_pool() : pool_size(std::thread::hardware_concurrency()), task_submitters(new tbb::task_group[pool_size]) {}
2987c2fdc1SPavel Kumbrasev
~task_group_pooltask_group_pool3087c2fdc1SPavel Kumbrasev ~task_group_pool() {
3187c2fdc1SPavel Kumbrasev for (std::size_t i = 0; i < pool_size; ++i) {
3287c2fdc1SPavel Kumbrasev task_submitters[i].wait();
3387c2fdc1SPavel Kumbrasev }
3487c2fdc1SPavel Kumbrasev
3587c2fdc1SPavel Kumbrasev delete [] task_submitters;
3687c2fdc1SPavel Kumbrasev }
3787c2fdc1SPavel Kumbrasev
3887c2fdc1SPavel Kumbrasev tbb::task_group& operator[] (std::size_t idx) { return task_submitters[idx]; }
3987c2fdc1SPavel Kumbrasev
4087c2fdc1SPavel Kumbrasev const std::size_t pool_size;
4187c2fdc1SPavel Kumbrasev tbb::task_group* task_submitters;
4287c2fdc1SPavel Kumbrasev };
4387c2fdc1SPavel Kumbrasev
4487c2fdc1SPavel Kumbrasev static task_group_pool tg_pool;
4587c2fdc1SPavel Kumbrasev
4687c2fdc1SPavel Kumbrasev class base_task {
4787c2fdc1SPavel Kumbrasev public:
4887c2fdc1SPavel Kumbrasev base_task() = default;
4987c2fdc1SPavel Kumbrasev
base_task(const base_task & t)50*91ba7e7eSPavel Kumbrasev base_task(const base_task& t) : m_type(t.m_type), m_parent(t.m_parent), m_child_counter(t.m_child_counter.load())
5187c2fdc1SPavel Kumbrasev {}
5287c2fdc1SPavel Kumbrasev
5387c2fdc1SPavel Kumbrasev virtual ~base_task() = default;
5487c2fdc1SPavel Kumbrasev
operator()5587c2fdc1SPavel Kumbrasev void operator() () const {
56*91ba7e7eSPavel Kumbrasev task_type type_snapshot = m_type;
57*91ba7e7eSPavel Kumbrasev
58*91ba7e7eSPavel Kumbrasev base_task* bypass = const_cast<base_task*>(this)->execute();
59*91ba7e7eSPavel Kumbrasev
60*91ba7e7eSPavel Kumbrasev if (m_parent && m_type != task_type::recycled) {
61*91ba7e7eSPavel Kumbrasev if (m_parent->remove_child_reference() == 0) {
62d489c1f7SPavel Kumbrasev m_parent->operator()();
63d489c1f7SPavel Kumbrasev }
64d489c1f7SPavel Kumbrasev }
65d489c1f7SPavel Kumbrasev
66*91ba7e7eSPavel Kumbrasev if (m_type == task_type::allocated) {
67d489c1f7SPavel Kumbrasev delete this;
6887c2fdc1SPavel Kumbrasev }
69*91ba7e7eSPavel Kumbrasev
70*91ba7e7eSPavel Kumbrasev if (bypass != nullptr) {
71*91ba7e7eSPavel Kumbrasev m_type = type_snapshot;
72*91ba7e7eSPavel Kumbrasev
73*91ba7e7eSPavel Kumbrasev // Bypass is not supported by task_emulation and next_task executed directly.
74*91ba7e7eSPavel Kumbrasev // However, the old-TBB bypass behavior can be achieved with
75*91ba7e7eSPavel Kumbrasev // `return task_group::defer()` (check Migration Guide).
76*91ba7e7eSPavel Kumbrasev // Consider submit another task if recursion call is not acceptable
77*91ba7e7eSPavel Kumbrasev // i.e. instead of Direct Body call
78*91ba7e7eSPavel Kumbrasev // submit task_emulation::run_task();
79*91ba7e7eSPavel Kumbrasev bypass->operator()();
80*91ba7e7eSPavel Kumbrasev }
8187c2fdc1SPavel Kumbrasev }
8287c2fdc1SPavel Kumbrasev
83*91ba7e7eSPavel Kumbrasev virtual base_task* execute() = 0;
8487c2fdc1SPavel Kumbrasev
8587c2fdc1SPavel Kumbrasev template <typename C, typename... Args>
allocate_continuation(std::uint64_t ref,Args &&...args)86d489c1f7SPavel Kumbrasev C* allocate_continuation(std::uint64_t ref, Args&&... args) {
8787c2fdc1SPavel Kumbrasev C* continuation = new C{std::forward<Args>(args)...};
88*91ba7e7eSPavel Kumbrasev continuation->m_type = task_type::allocated;
8987c2fdc1SPavel Kumbrasev continuation->reset_parent(reset_parent());
9087c2fdc1SPavel Kumbrasev continuation->m_child_counter = ref;
9187c2fdc1SPavel Kumbrasev return continuation;
9287c2fdc1SPavel Kumbrasev }
9387c2fdc1SPavel Kumbrasev
9487c2fdc1SPavel Kumbrasev template <typename F, typename... Args>
create_child(Args &&...args)95d489c1f7SPavel Kumbrasev F create_child(Args&&... args) {
96d489c1f7SPavel Kumbrasev return create_child_impl<F>(std::forward<Args>(args)...);
97d489c1f7SPavel Kumbrasev }
98d489c1f7SPavel Kumbrasev
99d489c1f7SPavel Kumbrasev template <typename F, typename... Args>
create_child_and_increment(Args &&...args)100d489c1f7SPavel Kumbrasev F create_child_and_increment(Args&&... args) {
101*91ba7e7eSPavel Kumbrasev add_child_reference();
102d489c1f7SPavel Kumbrasev return create_child_impl<F>(std::forward<Args>(args)...);
103d489c1f7SPavel Kumbrasev }
104d489c1f7SPavel Kumbrasev
105d489c1f7SPavel Kumbrasev template <typename F, typename... Args>
allocate_child(Args &&...args)106d489c1f7SPavel Kumbrasev F* allocate_child(Args&&... args) {
107d489c1f7SPavel Kumbrasev return allocate_child_impl<F>(std::forward<Args>(args)...);
108d489c1f7SPavel Kumbrasev }
109d489c1f7SPavel Kumbrasev
110d489c1f7SPavel Kumbrasev template <typename F, typename... Args>
allocate_child_and_increment(Args &&...args)111d489c1f7SPavel Kumbrasev F* allocate_child_and_increment(Args&&... args) {
112*91ba7e7eSPavel Kumbrasev add_child_reference();
113d489c1f7SPavel Kumbrasev return allocate_child_impl<F>(std::forward<Args>(args)...);
11487c2fdc1SPavel Kumbrasev }
11587c2fdc1SPavel Kumbrasev
11687c2fdc1SPavel Kumbrasev template <typename C>
recycle_as_child_of(C & c)117d489c1f7SPavel Kumbrasev void recycle_as_child_of(C& c) {
118*91ba7e7eSPavel Kumbrasev m_type = task_type::recycled;
11987c2fdc1SPavel Kumbrasev reset_parent(&c);
12087c2fdc1SPavel Kumbrasev }
12187c2fdc1SPavel Kumbrasev
recycle_as_continuation()122d489c1f7SPavel Kumbrasev void recycle_as_continuation() {
123*91ba7e7eSPavel Kumbrasev m_type = task_type::recycled;
124d489c1f7SPavel Kumbrasev }
125d489c1f7SPavel Kumbrasev
add_child_reference()126*91ba7e7eSPavel Kumbrasev void add_child_reference() {
12787c2fdc1SPavel Kumbrasev ++m_child_counter;
12887c2fdc1SPavel Kumbrasev }
12987c2fdc1SPavel Kumbrasev
remove_child_reference()130*91ba7e7eSPavel Kumbrasev std::uint64_t remove_child_reference() {
131d489c1f7SPavel Kumbrasev return --m_child_counter;
132d489c1f7SPavel Kumbrasev }
133d489c1f7SPavel Kumbrasev
134d489c1f7SPavel Kumbrasev protected:
135d489c1f7SPavel Kumbrasev enum class task_type {
136*91ba7e7eSPavel Kumbrasev stack_based,
137d489c1f7SPavel Kumbrasev allocated,
138*91ba7e7eSPavel Kumbrasev recycled
139d489c1f7SPavel Kumbrasev };
140d489c1f7SPavel Kumbrasev
141*91ba7e7eSPavel Kumbrasev mutable task_type m_type;
142d489c1f7SPavel Kumbrasev
14387c2fdc1SPavel Kumbrasev private:
14487c2fdc1SPavel Kumbrasev template <typename F, typename... Args>
14587c2fdc1SPavel Kumbrasev friend F create_root_task(tbb::task_group& tg, Args&&... args);
14687c2fdc1SPavel Kumbrasev
147d489c1f7SPavel Kumbrasev template <typename F, typename... Args>
148d489c1f7SPavel Kumbrasev friend F* allocate_root_task(tbb::task_group& tg, Args&&... args);
14987c2fdc1SPavel Kumbrasev
150d489c1f7SPavel Kumbrasev template <typename F, typename... Args>
create_child_impl(Args &&...args)151d489c1f7SPavel Kumbrasev F create_child_impl(Args&&... args) {
152d489c1f7SPavel Kumbrasev F obj{std::forward<Args>(args)...};
153*91ba7e7eSPavel Kumbrasev obj.m_type = task_type::stack_based;
154d489c1f7SPavel Kumbrasev obj.reset_parent(this);
155d489c1f7SPavel Kumbrasev return obj;
15687c2fdc1SPavel Kumbrasev }
157d489c1f7SPavel Kumbrasev
158d489c1f7SPavel Kumbrasev template <typename F, typename... Args>
allocate_child_impl(Args &&...args)159d489c1f7SPavel Kumbrasev F* allocate_child_impl(Args&&... args) {
160d489c1f7SPavel Kumbrasev F* obj = new F{std::forward<Args>(args)...};
161d489c1f7SPavel Kumbrasev obj->m_type = task_type::allocated;
162d489c1f7SPavel Kumbrasev obj->reset_parent(this);
163d489c1f7SPavel Kumbrasev return obj;
16487c2fdc1SPavel Kumbrasev }
16587c2fdc1SPavel Kumbrasev
16687c2fdc1SPavel Kumbrasev base_task* reset_parent(base_task* ptr = nullptr) {
16787c2fdc1SPavel Kumbrasev auto p = m_parent;
16887c2fdc1SPavel Kumbrasev m_parent = ptr;
16987c2fdc1SPavel Kumbrasev return p;
17087c2fdc1SPavel Kumbrasev }
17187c2fdc1SPavel Kumbrasev
17287c2fdc1SPavel Kumbrasev base_task* m_parent{nullptr};
17387c2fdc1SPavel Kumbrasev std::atomic<std::uint64_t> m_child_counter{0};
17487c2fdc1SPavel Kumbrasev };
17587c2fdc1SPavel Kumbrasev
17687c2fdc1SPavel Kumbrasev class root_task : public base_task {
17787c2fdc1SPavel Kumbrasev public:
root_task(tbb::task_group & tg)17887c2fdc1SPavel Kumbrasev root_task(tbb::task_group& tg) : m_tg(tg), m_callback(m_tg.defer([] { /* Create empty callback to preserve reference for wait. */})) {
179*91ba7e7eSPavel Kumbrasev add_child_reference();
180*91ba7e7eSPavel Kumbrasev m_type = base_task::task_type::allocated;
18187c2fdc1SPavel Kumbrasev }
18287c2fdc1SPavel Kumbrasev
18387c2fdc1SPavel Kumbrasev private:
execute()184*91ba7e7eSPavel Kumbrasev base_task* execute() override {
18587c2fdc1SPavel Kumbrasev m_tg.run(std::move(m_callback));
186*91ba7e7eSPavel Kumbrasev return nullptr;
18787c2fdc1SPavel Kumbrasev }
18887c2fdc1SPavel Kumbrasev
18987c2fdc1SPavel Kumbrasev tbb::task_group& m_tg;
19087c2fdc1SPavel Kumbrasev tbb::task_handle m_callback;
19187c2fdc1SPavel Kumbrasev };
19287c2fdc1SPavel Kumbrasev
19387c2fdc1SPavel Kumbrasev template <typename F, typename... Args>
create_root_task(tbb::task_group & tg,Args &&...args)19487c2fdc1SPavel Kumbrasev F create_root_task(tbb::task_group& tg, Args&&... args) {
19587c2fdc1SPavel Kumbrasev F obj{std::forward<Args>(args)...};
196*91ba7e7eSPavel Kumbrasev obj.m_type = base_task::task_type::stack_based;
19787c2fdc1SPavel Kumbrasev obj.reset_parent(new root_task{tg});
19887c2fdc1SPavel Kumbrasev return obj;
19987c2fdc1SPavel Kumbrasev }
20087c2fdc1SPavel Kumbrasev
201d489c1f7SPavel Kumbrasev template <typename F, typename... Args>
allocate_root_task(tbb::task_group & tg,Args &&...args)202d489c1f7SPavel Kumbrasev F* allocate_root_task(tbb::task_group& tg, Args&&... args) {
203d489c1f7SPavel Kumbrasev F* obj = new F{std::forward<Args>(args)...};
204d489c1f7SPavel Kumbrasev obj->m_type = base_task::task_type::allocated;
205d489c1f7SPavel Kumbrasev obj->reset_parent(new root_task{tg});
206d489c1f7SPavel Kumbrasev return obj;
207d489c1f7SPavel Kumbrasev }
208d489c1f7SPavel Kumbrasev
20987c2fdc1SPavel Kumbrasev template <typename F>
run_task(F && f)21087c2fdc1SPavel Kumbrasev void run_task(F&& f) {
21187c2fdc1SPavel Kumbrasev tg_pool[tbb::this_task_arena::current_thread_index()].run(std::forward<F>(f));
21287c2fdc1SPavel Kumbrasev }
213d489c1f7SPavel Kumbrasev
214d489c1f7SPavel Kumbrasev template <typename F>
run_task(F * f)215d489c1f7SPavel Kumbrasev void run_task(F* f) {
216d489c1f7SPavel Kumbrasev tg_pool[tbb::this_task_arena::current_thread_index()].run(std::ref(*f));
217d489c1f7SPavel Kumbrasev }
218d489c1f7SPavel Kumbrasev
219d489c1f7SPavel Kumbrasev template <typename F>
run_and_wait(tbb::task_group & tg,F * f)220d489c1f7SPavel Kumbrasev void run_and_wait(tbb::task_group& tg, F* f) {
221d489c1f7SPavel Kumbrasev tg.run_and_wait(std::ref(*f));
222d489c1f7SPavel Kumbrasev }
22387c2fdc1SPavel Kumbrasev } // namespace task_emulation
22487c2fdc1SPavel Kumbrasev
22587c2fdc1SPavel Kumbrasev #endif // __TBB_task_emulation_layer_H
226