149e08aacStbbdev /*
2e6e493f9SChristian Fersch     Copyright (c) 2005-2023 Intel Corporation
349e08aacStbbdev 
449e08aacStbbdev     Licensed under the Apache License, Version 2.0 (the "License");
549e08aacStbbdev     you may not use this file except in compliance with the License.
649e08aacStbbdev     You may obtain a copy of the License at
749e08aacStbbdev 
849e08aacStbbdev         http://www.apache.org/licenses/LICENSE-2.0
949e08aacStbbdev 
1049e08aacStbbdev     Unless required by applicable law or agreed to in writing, software
1149e08aacStbbdev     distributed under the License is distributed on an "AS IS" BASIS,
1249e08aacStbbdev     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1349e08aacStbbdev     See the License for the specific language governing permissions and
1449e08aacStbbdev     limitations under the License.
1549e08aacStbbdev */
1649e08aacStbbdev 
1749e08aacStbbdev #ifndef __TBB_parallel_for_each_H
1849e08aacStbbdev #define __TBB_parallel_for_each_H
1949e08aacStbbdev 
2049e08aacStbbdev #include "detail/_config.h"
2149e08aacStbbdev #include "detail/_namespace_injection.h"
2249e08aacStbbdev #include "detail/_exception.h"
2349e08aacStbbdev #include "detail/_task.h"
2449e08aacStbbdev #include "detail/_aligned_space.h"
2549e08aacStbbdev #include "detail/_small_object_pool.h"
26*a088cfa0SKonstantin Boyarinov #include "detail/_utils.h"
2749e08aacStbbdev 
2849e08aacStbbdev #include "parallel_for.h"
2949e08aacStbbdev #include "task_group.h" // task_group_context
3049e08aacStbbdev 
3149e08aacStbbdev #include <iterator>
3249e08aacStbbdev #include <type_traits>
3349e08aacStbbdev 
3449e08aacStbbdev namespace tbb {
3549e08aacStbbdev namespace detail {
36478de5b1Stbbdev #if __TBB_CPP20_CONCEPTS_PRESENT
37478de5b1Stbbdev namespace d1 {
38478de5b1Stbbdev template <typename Item>
39478de5b1Stbbdev class feeder;
40478de5b1Stbbdev 
41478de5b1Stbbdev } // namespace d1
42478de5b1Stbbdev inline namespace d0 {
43478de5b1Stbbdev 
44478de5b1Stbbdev template <typename Body, typename ItemType, typename FeederItemType>
45*a088cfa0SKonstantin Boyarinov concept parallel_for_each_body = std::invocable<const std::remove_reference_t<Body>&, ItemType&&> ||
46*a088cfa0SKonstantin Boyarinov                                  std::invocable<const std::remove_reference_t<Body>&, ItemType&&, tbb::detail::d1::feeder<FeederItemType>&>;
47478de5b1Stbbdev 
48478de5b1Stbbdev } // namespace d0
49478de5b1Stbbdev #endif // __TBB_CPP20_CONCEPTS_PRESENT
50b15aabb3Stbbdev namespace d2 {
5149e08aacStbbdev template<typename Body, typename Item> class feeder_impl;
52b15aabb3Stbbdev } // namespace d2
5349e08aacStbbdev 
54b15aabb3Stbbdev namespace d1 {
5549e08aacStbbdev //! Class the user supplied algorithm body uses to add new tasks
5649e08aacStbbdev template<typename Item>
5749e08aacStbbdev class feeder {
5849e08aacStbbdev     feeder() {}
5949e08aacStbbdev     feeder(const feeder&) = delete;
6049e08aacStbbdev     void operator=( const feeder&) = delete;
6149e08aacStbbdev 
6249e08aacStbbdev     virtual ~feeder () {}
6349e08aacStbbdev     virtual void internal_add_copy(const Item& item) = 0;
6449e08aacStbbdev     virtual void internal_add_move(Item&& item) = 0;
6549e08aacStbbdev 
66478de5b1Stbbdev     template<typename Body_, typename Item_> friend class d2::feeder_impl;
6749e08aacStbbdev public:
6849e08aacStbbdev     //! Add a work item to a running parallel_for_each.
6949e08aacStbbdev     void add(const Item& item) {internal_add_copy(item);}
7049e08aacStbbdev     void add(Item&& item) {internal_add_move(std::move(item));}
7149e08aacStbbdev };
7249e08aacStbbdev 
73b15aabb3Stbbdev } // namespace d1
74b15aabb3Stbbdev 
75b15aabb3Stbbdev namespace d2 {
76b15aabb3Stbbdev using namespace tbb::detail::d1;
7749e08aacStbbdev /** Selects one of the two possible forms of function call member operator.
7849e08aacStbbdev     @ingroup algorithms **/
79b15aabb3Stbbdev template<class Body>
80b15aabb3Stbbdev struct parallel_for_each_operator_selector {
8149e08aacStbbdev public:
8249e08aacStbbdev     template<typename ItemArg, typename FeederArg>
83b15aabb3Stbbdev     static auto call(const Body& body, ItemArg&& item, FeederArg*)
84*a088cfa0SKonstantin Boyarinov     -> decltype(tbb::detail::invoke(body, std::forward<ItemArg>(item)), void()) {
8549e08aacStbbdev         #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
8649e08aacStbbdev         // Suppression of Microsoft non-standard extension warnings
8749e08aacStbbdev         #pragma warning (push)
8849e08aacStbbdev         #pragma warning (disable: 4239)
8949e08aacStbbdev         #endif
9049e08aacStbbdev 
91*a088cfa0SKonstantin Boyarinov         tbb::detail::invoke(body, std::forward<ItemArg>(item));
9249e08aacStbbdev 
9349e08aacStbbdev         #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
94e6e493f9SChristian Fersch         #pragma warning (pop)
9549e08aacStbbdev         #endif
9649e08aacStbbdev     }
9749e08aacStbbdev 
9849e08aacStbbdev     template<typename ItemArg, typename FeederArg>
99b15aabb3Stbbdev     static auto call(const Body& body, ItemArg&& item, FeederArg* feeder)
100*a088cfa0SKonstantin Boyarinov     -> decltype(tbb::detail::invoke(body, std::forward<ItemArg>(item), *feeder), void()) {
10149e08aacStbbdev         #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
10249e08aacStbbdev         // Suppression of Microsoft non-standard extension warnings
10349e08aacStbbdev         #pragma warning (push)
10449e08aacStbbdev         #pragma warning (disable: 4239)
10549e08aacStbbdev         #endif
106b15aabb3Stbbdev         __TBB_ASSERT(feeder, "Feeder was not created but should be");
10749e08aacStbbdev 
108*a088cfa0SKonstantin Boyarinov         tbb::detail::invoke(body, std::forward<ItemArg>(item), *feeder);
10949e08aacStbbdev 
11049e08aacStbbdev         #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
111e6e493f9SChristian Fersch         #pragma warning (pop)
11249e08aacStbbdev         #endif
11349e08aacStbbdev     }
11449e08aacStbbdev };
11549e08aacStbbdev 
11649e08aacStbbdev template<typename Body, typename Item>
11749e08aacStbbdev struct feeder_item_task: public task {
11849e08aacStbbdev     using feeder_type = feeder_impl<Body, Item>;
11949e08aacStbbdev 
12049e08aacStbbdev     template <typename ItemType>
12149e08aacStbbdev     feeder_item_task(ItemType&& input_item, feeder_type& feeder, small_object_allocator& alloc) :
12249e08aacStbbdev         item(std::forward<ItemType>(input_item)),
12349e08aacStbbdev         my_feeder(feeder),
12449e08aacStbbdev         my_allocator(alloc)
12549e08aacStbbdev     {}
12649e08aacStbbdev 
12749e08aacStbbdev     void finalize(const execution_data& ed) {
12849e08aacStbbdev         my_feeder.my_wait_context.release();
12949e08aacStbbdev         my_allocator.delete_object(this, ed);
13049e08aacStbbdev     }
13149e08aacStbbdev 
132b15aabb3Stbbdev     //! Hack for resolve ambiguity between calls to the body with and without moving the stored copy
133b15aabb3Stbbdev     //! Executing body with moving the copy should have higher priority
134b15aabb3Stbbdev     using first_priority = int;
135b15aabb3Stbbdev     using second_priority = double;
136b15aabb3Stbbdev 
137b15aabb3Stbbdev     template <typename BodyType, typename ItemType, typename FeederType>
138b15aabb3Stbbdev     static auto call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, first_priority)
139b15aabb3Stbbdev     -> decltype(parallel_for_each_operator_selector<Body>::call(call_body, std::move(call_item), &call_feeder), void())
140b15aabb3Stbbdev     {
141b15aabb3Stbbdev         parallel_for_each_operator_selector<Body>::call(call_body, std::move(call_item), &call_feeder);
142b15aabb3Stbbdev     }
143b15aabb3Stbbdev 
144b15aabb3Stbbdev     template <typename BodyType, typename ItemType, typename FeederType>
145b15aabb3Stbbdev     static void call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, second_priority) {
146b15aabb3Stbbdev         parallel_for_each_operator_selector<Body>::call(call_body, call_item, &call_feeder);
147b15aabb3Stbbdev     }
148b15aabb3Stbbdev 
14949e08aacStbbdev     task* execute(execution_data& ed) override {
150b15aabb3Stbbdev         call(my_feeder.my_body, item, my_feeder, first_priority{});
15149e08aacStbbdev         finalize(ed);
15249e08aacStbbdev         return nullptr;
15349e08aacStbbdev     }
15449e08aacStbbdev 
15549e08aacStbbdev     task* cancel(execution_data& ed) override {
15649e08aacStbbdev         finalize(ed);
15749e08aacStbbdev         return nullptr;
15849e08aacStbbdev     }
15949e08aacStbbdev 
16049e08aacStbbdev     Item item;
16149e08aacStbbdev     feeder_type& my_feeder;
16249e08aacStbbdev     small_object_allocator my_allocator;
16349e08aacStbbdev }; // class feeder_item_task
16449e08aacStbbdev 
16549e08aacStbbdev /** Implements new task adding procedure.
16649e08aacStbbdev     @ingroup algorithms **/
167b15aabb3Stbbdev template<typename Body, typename Item>
16849e08aacStbbdev class feeder_impl : public feeder<Item> {
16949e08aacStbbdev     // Avoiding use of copy constructor in a virtual method if the type does not support it
17049e08aacStbbdev     void internal_add_copy_impl(std::true_type, const Item& item) {
17149e08aacStbbdev         using feeder_task = feeder_item_task<Body, Item>;
17249e08aacStbbdev         small_object_allocator alloc;
17349e08aacStbbdev         auto task = alloc.new_object<feeder_task>(item, *this, alloc);
17449e08aacStbbdev 
17549e08aacStbbdev         my_wait_context.reserve();
17649e08aacStbbdev         spawn(*task, my_execution_context);
17749e08aacStbbdev     }
17849e08aacStbbdev 
17949e08aacStbbdev     void internal_add_copy_impl(std::false_type, const Item&) {
18049e08aacStbbdev         __TBB_ASSERT(false, "Overloading for r-value reference doesn't work or it's not movable and not copyable object");
18149e08aacStbbdev     }
18249e08aacStbbdev 
18349e08aacStbbdev     void internal_add_copy(const Item& item) override {
18449e08aacStbbdev         internal_add_copy_impl(typename std::is_copy_constructible<Item>::type(), item);
18549e08aacStbbdev     }
18649e08aacStbbdev 
18749e08aacStbbdev     void internal_add_move(Item&& item) override {
18849e08aacStbbdev         using feeder_task = feeder_item_task<Body, Item>;
18949e08aacStbbdev         small_object_allocator alloc{};
19049e08aacStbbdev         auto task = alloc.new_object<feeder_task>(std::move(item), *this, alloc);
19149e08aacStbbdev 
19249e08aacStbbdev         my_wait_context.reserve();
19349e08aacStbbdev         spawn(*task, my_execution_context);
19449e08aacStbbdev     }
19549e08aacStbbdev public:
196b15aabb3Stbbdev     feeder_impl(const Body& body, wait_context& w_context, task_group_context &context)
197b15aabb3Stbbdev       : my_body(body),
198b15aabb3Stbbdev         my_wait_context(w_context)
19949e08aacStbbdev       , my_execution_context(context)
20049e08aacStbbdev     {}
20149e08aacStbbdev 
20249e08aacStbbdev     const Body& my_body;
203b15aabb3Stbbdev     wait_context& my_wait_context;
20449e08aacStbbdev     task_group_context& my_execution_context;
20549e08aacStbbdev }; // class feeder_impl
20649e08aacStbbdev 
20749e08aacStbbdev /** Execute computation under one element of the range
20849e08aacStbbdev     @ingroup algorithms **/
20949e08aacStbbdev template<typename Iterator, typename Body, typename Item>
21049e08aacStbbdev struct for_each_iteration_task: public task {
21149e08aacStbbdev     using feeder_type = feeder_impl<Body, Item>;
21249e08aacStbbdev 
213b15aabb3Stbbdev     for_each_iteration_task(Iterator input_item_ptr, const Body& body, feeder_impl<Body, Item>* feeder_ptr, wait_context& wait_context) :
214b15aabb3Stbbdev         item_ptr(input_item_ptr), my_body(body), my_feeder_ptr(feeder_ptr), parent_wait_context(wait_context)
21549e08aacStbbdev     {}
21649e08aacStbbdev 
21749e08aacStbbdev     void finalize() {
21849e08aacStbbdev         parent_wait_context.release();
21949e08aacStbbdev     }
22049e08aacStbbdev 
22149e08aacStbbdev     task* execute(execution_data&) override {
222b15aabb3Stbbdev         parallel_for_each_operator_selector<Body>::call(my_body, *item_ptr, my_feeder_ptr);
22349e08aacStbbdev         finalize();
22449e08aacStbbdev         return nullptr;
22549e08aacStbbdev     }
22649e08aacStbbdev 
22749e08aacStbbdev     task* cancel(execution_data&) override {
22849e08aacStbbdev         finalize();
22949e08aacStbbdev         return nullptr;
23049e08aacStbbdev     }
23149e08aacStbbdev 
23249e08aacStbbdev     Iterator item_ptr;
233b15aabb3Stbbdev     const Body& my_body;
234b15aabb3Stbbdev     feeder_impl<Body, Item>* my_feeder_ptr;
23549e08aacStbbdev     wait_context& parent_wait_context;
236b15aabb3Stbbdev }; // class for_each_iteration_task
237b15aabb3Stbbdev 
238b15aabb3Stbbdev // Helper to get the type of the iterator to the internal sequence of copies
239b15aabb3Stbbdev // If the element can be passed to the body as an rvalue - this iterator should be move_iterator
240b15aabb3Stbbdev template <typename Body, typename Item, typename = void>
241b15aabb3Stbbdev struct input_iteration_task_iterator_helper {
242b15aabb3Stbbdev     // For input iterators we pass const lvalue reference to the body
243b15aabb3Stbbdev     // It is prohibited to take non-constant lvalue references for input iterators
244b15aabb3Stbbdev     using type = const Item*;
245b15aabb3Stbbdev };
246b15aabb3Stbbdev 
247b15aabb3Stbbdev template <typename Body, typename Item>
248b15aabb3Stbbdev struct input_iteration_task_iterator_helper<Body, Item,
249b15aabb3Stbbdev     tbb::detail::void_t<decltype(parallel_for_each_operator_selector<Body>::call(std::declval<const Body&>(),
250b15aabb3Stbbdev                                                                                  std::declval<Item&&>(),
251b15aabb3Stbbdev                                                                                  std::declval<feeder_impl<Body, Item>*>()))>>
252b15aabb3Stbbdev {
253b15aabb3Stbbdev     using type = std::move_iterator<Item*>;
254b15aabb3Stbbdev };
25549e08aacStbbdev 
25649e08aacStbbdev /** Split one block task to several(max_block_size) iteration tasks for input iterators
25749e08aacStbbdev     @ingroup algorithms **/
25849e08aacStbbdev template <typename Body, typename Item>
25949e08aacStbbdev struct input_block_handling_task : public task {
26049e08aacStbbdev     static constexpr size_t max_block_size = 4;
26149e08aacStbbdev 
26249e08aacStbbdev     using feeder_type = feeder_impl<Body, Item>;
263b15aabb3Stbbdev     using iteration_task_iterator_type = typename input_iteration_task_iterator_helper<Body, Item>::type;
264b15aabb3Stbbdev     using iteration_task = for_each_iteration_task<iteration_task_iterator_type, Body, Item>;
26549e08aacStbbdev 
266b15aabb3Stbbdev     input_block_handling_task(wait_context& root_wait_context, task_group_context& e_context,
267b15aabb3Stbbdev                               const Body& body, feeder_impl<Body, Item>* feeder_ptr, small_object_allocator& alloc)
268b15aabb3Stbbdev         :my_size(0), my_wait_context(0), my_root_wait_context(root_wait_context),
269b15aabb3Stbbdev          my_execution_context(e_context), my_allocator(alloc)
27049e08aacStbbdev     {
271b15aabb3Stbbdev         auto item_it = block_iteration_space.begin();
27249e08aacStbbdev         for (auto* it = task_pool.begin(); it != task_pool.end(); ++it) {
273b15aabb3Stbbdev             new (it) iteration_task(iteration_task_iterator_type(item_it++), body, feeder_ptr, my_wait_context);
27449e08aacStbbdev         }
27549e08aacStbbdev     }
27649e08aacStbbdev 
27749e08aacStbbdev     void finalize(const execution_data& ed) {
278b15aabb3Stbbdev         my_root_wait_context.release();
27949e08aacStbbdev         my_allocator.delete_object(this, ed);
28049e08aacStbbdev     }
28149e08aacStbbdev 
28249e08aacStbbdev     task* execute(execution_data& ed) override {
28349e08aacStbbdev         __TBB_ASSERT( my_size > 0, "Negative size was passed to task");
28449e08aacStbbdev         for (std::size_t counter = 1; counter < my_size; ++counter) {
28549e08aacStbbdev             my_wait_context.reserve();
286b15aabb3Stbbdev             spawn(*(task_pool.begin() + counter), my_execution_context);
28749e08aacStbbdev         }
28849e08aacStbbdev         my_wait_context.reserve();
289b15aabb3Stbbdev         execute_and_wait(*task_pool.begin(), my_execution_context,
290b15aabb3Stbbdev                          my_wait_context,    my_execution_context);
29149e08aacStbbdev 
29249e08aacStbbdev         // deallocate current task after children execution
29349e08aacStbbdev         finalize(ed);
29449e08aacStbbdev         return nullptr;
29549e08aacStbbdev     }
29649e08aacStbbdev 
29749e08aacStbbdev     task* cancel(execution_data& ed) override {
29849e08aacStbbdev         finalize(ed);
29949e08aacStbbdev         return nullptr;
30049e08aacStbbdev     }
30149e08aacStbbdev 
30249e08aacStbbdev     ~input_block_handling_task() {
30349e08aacStbbdev         for(std::size_t counter = 0; counter < max_block_size; ++counter) {
30449e08aacStbbdev             (task_pool.begin() + counter)->~iteration_task();
305aa247fa8SIlya Isaev             if (counter < my_size) {
30649e08aacStbbdev                 (block_iteration_space.begin() + counter)->~Item();
30749e08aacStbbdev             }
30849e08aacStbbdev         }
309aa247fa8SIlya Isaev     }
31049e08aacStbbdev 
31149e08aacStbbdev     aligned_space<Item, max_block_size> block_iteration_space;
31249e08aacStbbdev     aligned_space<iteration_task, max_block_size> task_pool;
31349e08aacStbbdev     std::size_t my_size;
31449e08aacStbbdev     wait_context my_wait_context;
315b15aabb3Stbbdev     wait_context& my_root_wait_context;
316b15aabb3Stbbdev     task_group_context& my_execution_context;
31749e08aacStbbdev     small_object_allocator my_allocator;
318b15aabb3Stbbdev }; // class input_block_handling_task
31949e08aacStbbdev 
32049e08aacStbbdev /** Split one block task to several(max_block_size) iteration tasks for forward iterators
32149e08aacStbbdev     @ingroup algorithms **/
32249e08aacStbbdev template <typename Iterator, typename Body, typename Item>
32349e08aacStbbdev struct forward_block_handling_task : public task {
32449e08aacStbbdev     static constexpr size_t max_block_size = 4;
32549e08aacStbbdev 
32649e08aacStbbdev     using iteration_task = for_each_iteration_task<Iterator, Body, Item>;
32749e08aacStbbdev 
328b15aabb3Stbbdev     forward_block_handling_task(Iterator first, std::size_t size,
329b15aabb3Stbbdev                                 wait_context& w_context, task_group_context& e_context,
330b15aabb3Stbbdev                                 const Body& body, feeder_impl<Body, Item>* feeder_ptr,
331b15aabb3Stbbdev                                 small_object_allocator& alloc)
332b15aabb3Stbbdev         : my_size(size), my_wait_context(0), my_root_wait_context(w_context),
333b15aabb3Stbbdev           my_execution_context(e_context), my_allocator(alloc)
33449e08aacStbbdev     {
33549e08aacStbbdev         auto* task_it = task_pool.begin();
33649e08aacStbbdev         for (std::size_t i = 0; i < size; i++) {
337b15aabb3Stbbdev             new (task_it++) iteration_task(first, body, feeder_ptr, my_wait_context);
33849e08aacStbbdev             ++first;
33949e08aacStbbdev         }
34049e08aacStbbdev     }
34149e08aacStbbdev 
34249e08aacStbbdev     void finalize(const execution_data& ed) {
343b15aabb3Stbbdev         my_root_wait_context.release();
34449e08aacStbbdev         my_allocator.delete_object(this, ed);
34549e08aacStbbdev     }
34649e08aacStbbdev 
34749e08aacStbbdev     task* execute(execution_data& ed) override {
34849e08aacStbbdev         __TBB_ASSERT( my_size > 0, "Negative size was passed to task");
34949e08aacStbbdev         for(std::size_t counter = 1; counter < my_size; ++counter) {
35049e08aacStbbdev             my_wait_context.reserve();
351b15aabb3Stbbdev             spawn(*(task_pool.begin() + counter), my_execution_context);
35249e08aacStbbdev         }
35349e08aacStbbdev         my_wait_context.reserve();
354b15aabb3Stbbdev         execute_and_wait(*task_pool.begin(), my_execution_context,
355b15aabb3Stbbdev                          my_wait_context,    my_execution_context);
35649e08aacStbbdev 
35749e08aacStbbdev         // deallocate current task after children execution
35849e08aacStbbdev         finalize(ed);
35949e08aacStbbdev         return nullptr;
36049e08aacStbbdev     }
36149e08aacStbbdev 
36249e08aacStbbdev     task* cancel(execution_data& ed) override {
36349e08aacStbbdev         finalize(ed);
36449e08aacStbbdev         return nullptr;
36549e08aacStbbdev     }
36649e08aacStbbdev 
36749e08aacStbbdev     ~forward_block_handling_task() {
36849e08aacStbbdev         for(std::size_t counter = 0; counter < my_size; ++counter) {
36949e08aacStbbdev             (task_pool.begin() + counter)->~iteration_task();
37049e08aacStbbdev         }
37149e08aacStbbdev     }
37249e08aacStbbdev 
37349e08aacStbbdev     aligned_space<iteration_task, max_block_size> task_pool;
37449e08aacStbbdev     std::size_t my_size;
37549e08aacStbbdev     wait_context my_wait_context;
376b15aabb3Stbbdev     wait_context& my_root_wait_context;
377b15aabb3Stbbdev     task_group_context& my_execution_context;
37849e08aacStbbdev     small_object_allocator my_allocator;
37949e08aacStbbdev }; // class forward_block_handling_task
38049e08aacStbbdev 
38149e08aacStbbdev /** Body for parallel_for algorithm.
38249e08aacStbbdev   * Allows to redirect operations under random access iterators range to the parallel_for algorithm.
38349e08aacStbbdev     @ingroup algorithms **/
38449e08aacStbbdev template <typename Iterator, typename Body, typename Item>
38549e08aacStbbdev class parallel_for_body_wrapper {
38649e08aacStbbdev     Iterator my_first;
387b15aabb3Stbbdev     const Body& my_body;
388b15aabb3Stbbdev     feeder_impl<Body, Item>* my_feeder_ptr;
38949e08aacStbbdev public:
390b15aabb3Stbbdev     parallel_for_body_wrapper(Iterator first, const Body& body, feeder_impl<Body, Item>* feeder_ptr)
391b15aabb3Stbbdev         : my_first(first), my_body(body), my_feeder_ptr(feeder_ptr) {}
39249e08aacStbbdev 
39349e08aacStbbdev     void operator()(tbb::blocked_range<std::size_t> range) const {
39449e08aacStbbdev #if __INTEL_COMPILER
39549e08aacStbbdev #pragma ivdep
39649e08aacStbbdev #endif
39749e08aacStbbdev         for (std::size_t count = range.begin(); count != range.end(); count++) {
398b15aabb3Stbbdev             parallel_for_each_operator_selector<Body>::call(my_body, *(my_first + count),
399b15aabb3Stbbdev                                                             my_feeder_ptr);
40049e08aacStbbdev         }
40149e08aacStbbdev     }
40249e08aacStbbdev }; // class parallel_for_body_wrapper
40349e08aacStbbdev 
40449e08aacStbbdev 
40549e08aacStbbdev /** Helper for getting iterators tag including inherited custom tags
40649e08aacStbbdev     @ingroup algorithms */
40749e08aacStbbdev template<typename It>
40849e08aacStbbdev using tag = typename std::iterator_traits<It>::iterator_category;
40949e08aacStbbdev 
41049e08aacStbbdev template<typename It>
41149e08aacStbbdev using iterator_tag_dispatch = typename
41249e08aacStbbdev     std::conditional<
41349e08aacStbbdev         std::is_base_of<std::random_access_iterator_tag, tag<It>>::value,
41449e08aacStbbdev         std::random_access_iterator_tag,
41549e08aacStbbdev         typename std::conditional<
41649e08aacStbbdev             std::is_base_of<std::forward_iterator_tag, tag<It>>::value,
41749e08aacStbbdev             std::forward_iterator_tag,
41849e08aacStbbdev             std::input_iterator_tag
41949e08aacStbbdev         >::type
42049e08aacStbbdev     >::type;
42149e08aacStbbdev 
422b15aabb3Stbbdev template <typename Body, typename Iterator, typename Item>
423*a088cfa0SKonstantin Boyarinov using feeder_is_required = tbb::detail::void_t<decltype(tbb::detail::invoke(std::declval<const Body>(),
424*a088cfa0SKonstantin Boyarinov                                                                             std::declval<typename std::iterator_traits<Iterator>::reference>(),
425b15aabb3Stbbdev                                                                             std::declval<feeder<Item>&>()))>;
426b15aabb3Stbbdev 
427b15aabb3Stbbdev // Creates feeder object only if the body can accept it
428b15aabb3Stbbdev template <typename Iterator, typename Body, typename Item, typename = void>
429b15aabb3Stbbdev struct feeder_holder {
430b15aabb3Stbbdev     feeder_holder( wait_context&, task_group_context&, const Body& ) {}
431b15aabb3Stbbdev 
432b15aabb3Stbbdev     feeder_impl<Body, Item>* feeder_ptr() { return nullptr; }
433b15aabb3Stbbdev }; // class feeder_holder
434b15aabb3Stbbdev 
435b15aabb3Stbbdev template <typename Iterator, typename Body, typename Item>
436b15aabb3Stbbdev class feeder_holder<Iterator, Body, Item, feeder_is_required<Body, Iterator, Item>> {
437b15aabb3Stbbdev public:
438b15aabb3Stbbdev     feeder_holder( wait_context& w_context, task_group_context& context, const Body& body )
439b15aabb3Stbbdev         : my_feeder(body, w_context, context) {}
440b15aabb3Stbbdev 
441b15aabb3Stbbdev     feeder_impl<Body, Item>* feeder_ptr() { return &my_feeder; }
442b15aabb3Stbbdev private:
443b15aabb3Stbbdev     feeder_impl<Body, Item> my_feeder;
444b15aabb3Stbbdev }; // class feeder_holder
445b15aabb3Stbbdev 
446b15aabb3Stbbdev template <typename Iterator, typename Body, typename Item>
447b15aabb3Stbbdev class for_each_root_task_base : public task {
448b15aabb3Stbbdev public:
449b15aabb3Stbbdev     for_each_root_task_base(Iterator first, Iterator last, const Body& body, wait_context& w_context, task_group_context& e_context)
450b15aabb3Stbbdev         : my_first(first), my_last(last), my_wait_context(w_context), my_execution_context(e_context),
451b15aabb3Stbbdev           my_body(body), my_feeder_holder(my_wait_context, my_execution_context, my_body)
452b15aabb3Stbbdev     {
453b15aabb3Stbbdev         my_wait_context.reserve();
454b15aabb3Stbbdev     }
455b15aabb3Stbbdev private:
456b15aabb3Stbbdev     task* cancel(execution_data&) override {
457b15aabb3Stbbdev         this->my_wait_context.release();
458b15aabb3Stbbdev         return nullptr;
459b15aabb3Stbbdev     }
460b15aabb3Stbbdev protected:
461b15aabb3Stbbdev     Iterator my_first;
462b15aabb3Stbbdev     Iterator my_last;
463b15aabb3Stbbdev     wait_context& my_wait_context;
464b15aabb3Stbbdev     task_group_context& my_execution_context;
465b15aabb3Stbbdev     const Body& my_body;
466b15aabb3Stbbdev     feeder_holder<Iterator, Body, Item> my_feeder_holder;
467b15aabb3Stbbdev }; // class for_each_root_task_base
468b15aabb3Stbbdev 
46949e08aacStbbdev /** parallel_for_each algorithm root task - most generic version
47049e08aacStbbdev   * Splits input range to blocks
47149e08aacStbbdev     @ingroup algorithms **/
47249e08aacStbbdev template <typename Iterator, typename Body, typename Item, typename IteratorTag = iterator_tag_dispatch<Iterator>>
473b15aabb3Stbbdev class for_each_root_task : public for_each_root_task_base<Iterator, Body, Item>
47449e08aacStbbdev {
475b15aabb3Stbbdev     using base_type = for_each_root_task_base<Iterator, Body, Item>;
476b15aabb3Stbbdev public:
477b15aabb3Stbbdev     using base_type::base_type;
47849e08aacStbbdev private:
47949e08aacStbbdev     task* execute(execution_data& ed) override {
48049e08aacStbbdev         using block_handling_type = input_block_handling_task<Body, Item>;
48149e08aacStbbdev 
482b15aabb3Stbbdev         if (this->my_first == this->my_last) {
483b15aabb3Stbbdev             this->my_wait_context.release();
48449e08aacStbbdev             return nullptr;
48549e08aacStbbdev         }
48649e08aacStbbdev 
487b15aabb3Stbbdev         this->my_wait_context.reserve();
48849e08aacStbbdev         small_object_allocator alloc{};
489b15aabb3Stbbdev         auto block_handling_task = alloc.new_object<block_handling_type>(ed, this->my_wait_context, this->my_execution_context,
490b15aabb3Stbbdev                                                                          this->my_body, this->my_feeder_holder.feeder_ptr(),
491b15aabb3Stbbdev                                                                          alloc);
49249e08aacStbbdev 
49349e08aacStbbdev         auto* block_iterator = block_handling_task->block_iteration_space.begin();
494b15aabb3Stbbdev         for (; !(this->my_first == this->my_last) && block_handling_task->my_size < block_handling_type::max_block_size; ++this->my_first) {
49549e08aacStbbdev             // Move semantics are automatically used when supported by the iterator
496b15aabb3Stbbdev             new (block_iterator++) Item(*this->my_first);
49749e08aacStbbdev             ++block_handling_task->my_size;
49849e08aacStbbdev         }
49949e08aacStbbdev 
50049e08aacStbbdev         // Do not access this after spawn to avoid races
501b15aabb3Stbbdev         spawn(*this, this->my_execution_context);
50249e08aacStbbdev         return block_handling_task;
50349e08aacStbbdev     }
50449e08aacStbbdev }; // class for_each_root_task - most generic implementation
50549e08aacStbbdev 
50649e08aacStbbdev /** parallel_for_each algorithm root task - forward iterator based specialization
50749e08aacStbbdev   * Splits input range to blocks
50849e08aacStbbdev     @ingroup algorithms **/
50949e08aacStbbdev template <typename Iterator, typename Body, typename Item>
510b15aabb3Stbbdev class for_each_root_task<Iterator, Body, Item, std::forward_iterator_tag>
511b15aabb3Stbbdev     : public for_each_root_task_base<Iterator, Body, Item>
51249e08aacStbbdev {
513b15aabb3Stbbdev     using base_type = for_each_root_task_base<Iterator, Body, Item>;
514b15aabb3Stbbdev public:
515b15aabb3Stbbdev     using base_type::base_type;
51649e08aacStbbdev private:
51749e08aacStbbdev     task* execute(execution_data& ed) override {
51849e08aacStbbdev         using block_handling_type = forward_block_handling_task<Iterator, Body, Item>;
519b15aabb3Stbbdev         if (this->my_first == this->my_last) {
520b15aabb3Stbbdev             this->my_wait_context.release();
52149e08aacStbbdev             return nullptr;
52249e08aacStbbdev         }
52349e08aacStbbdev 
52449e08aacStbbdev         std::size_t block_size{0};
525b15aabb3Stbbdev         Iterator first_block_element = this->my_first;
526b15aabb3Stbbdev         for (; !(this->my_first == this->my_last) && block_size < block_handling_type::max_block_size; ++this->my_first) {
52749e08aacStbbdev             ++block_size;
52849e08aacStbbdev         }
52949e08aacStbbdev 
530b15aabb3Stbbdev         this->my_wait_context.reserve();
53149e08aacStbbdev         small_object_allocator alloc{};
532b15aabb3Stbbdev         auto block_handling_task = alloc.new_object<block_handling_type>(ed, first_block_element, block_size,
533b15aabb3Stbbdev                                                                          this->my_wait_context, this->my_execution_context,
534b15aabb3Stbbdev                                                                          this->my_body, this->my_feeder_holder.feeder_ptr(), alloc);
53549e08aacStbbdev 
53649e08aacStbbdev         // Do not access this after spawn to avoid races
537b15aabb3Stbbdev         spawn(*this, this->my_execution_context);
53849e08aacStbbdev         return block_handling_task;
53949e08aacStbbdev     }
54049e08aacStbbdev }; // class for_each_root_task - forward iterator based specialization
54149e08aacStbbdev 
54249e08aacStbbdev /** parallel_for_each algorithm root task - random access iterator based specialization
54349e08aacStbbdev   * Splits input range to blocks
54449e08aacStbbdev     @ingroup algorithms **/
54549e08aacStbbdev template <typename Iterator, typename Body, typename Item>
546b15aabb3Stbbdev class for_each_root_task<Iterator, Body, Item, std::random_access_iterator_tag>
547b15aabb3Stbbdev     : public for_each_root_task_base<Iterator, Body, Item>
54849e08aacStbbdev {
549b15aabb3Stbbdev     using base_type = for_each_root_task_base<Iterator, Body, Item>;
550b15aabb3Stbbdev public:
551b15aabb3Stbbdev     using base_type::base_type;
55249e08aacStbbdev private:
55349e08aacStbbdev     task* execute(execution_data&) override {
55449e08aacStbbdev         tbb::parallel_for(
555b15aabb3Stbbdev             tbb::blocked_range<std::size_t>(0, std::distance(this->my_first, this->my_last)),
556b15aabb3Stbbdev             parallel_for_body_wrapper<Iterator, Body, Item>(this->my_first, this->my_body, this->my_feeder_holder.feeder_ptr())
557b15aabb3Stbbdev             , this->my_execution_context
55849e08aacStbbdev         );
55949e08aacStbbdev 
560b15aabb3Stbbdev         this->my_wait_context.release();
56149e08aacStbbdev         return nullptr;
56249e08aacStbbdev     }
56349e08aacStbbdev }; // class for_each_root_task - random access iterator based specialization
56449e08aacStbbdev 
56549e08aacStbbdev /** Helper for getting item type. If item type can be deduced from feeder - got it from feeder,
56649e08aacStbbdev     if feeder is generic - got item type from range.
56749e08aacStbbdev     @ingroup algorithms */
56849e08aacStbbdev template<typename Body, typename Item, typename FeederArg>
56949e08aacStbbdev auto feeder_argument_parser(void (Body::*)(Item, feeder<FeederArg>&) const) -> FeederArg;
57049e08aacStbbdev 
57149e08aacStbbdev template<typename Body, typename>
57249e08aacStbbdev decltype(feeder_argument_parser<Body>(&Body::operator())) get_item_type_impl(int); // for (T, feeder<T>)
57349e08aacStbbdev template<typename Body, typename Item> Item get_item_type_impl(...); // stub
57449e08aacStbbdev 
57549e08aacStbbdev template <typename Body, typename Item>
57649e08aacStbbdev using get_item_type = decltype(get_item_type_impl<Body, Item>(0));
57749e08aacStbbdev 
578478de5b1Stbbdev #if __TBB_CPP20_CONCEPTS_PRESENT
579478de5b1Stbbdev template <typename Body, typename ItemType>
580478de5b1Stbbdev using feeder_item_type = std::remove_cvref_t<get_item_type<Body, ItemType>>;
581478de5b1Stbbdev 
582478de5b1Stbbdev template <typename Body, typename Iterator>
583478de5b1Stbbdev concept parallel_for_each_iterator_body =
584478de5b1Stbbdev     parallel_for_each_body<Body, iterator_reference_type<Iterator>, feeder_item_type<Body, iterator_reference_type<Iterator>>>;
585478de5b1Stbbdev 
586478de5b1Stbbdev template <typename Body, typename Range>
587478de5b1Stbbdev concept parallel_for_each_range_body =
588478de5b1Stbbdev     parallel_for_each_body<Body, range_reference_type<Range>, feeder_item_type<Body, range_reference_type<Range>>>;
589478de5b1Stbbdev #endif
590478de5b1Stbbdev 
59149e08aacStbbdev /** Implements parallel iteration over a range.
59249e08aacStbbdev     @ingroup algorithms */
59349e08aacStbbdev template<typename Iterator, typename Body>
59449e08aacStbbdev void run_parallel_for_each( Iterator first, Iterator last, const Body& body, task_group_context& context)
59549e08aacStbbdev {
59649e08aacStbbdev     if (!(first == last)) {
597b15aabb3Stbbdev         using ItemType = get_item_type<Body, typename std::iterator_traits<Iterator>::value_type>;
598b15aabb3Stbbdev         wait_context w_context(0);
59949e08aacStbbdev 
600b15aabb3Stbbdev         for_each_root_task<Iterator, Body, ItemType> root_task(first, last, body, w_context, context);
60149e08aacStbbdev 
602b15aabb3Stbbdev         execute_and_wait(root_task, context, w_context, context);
60349e08aacStbbdev     }
60449e08aacStbbdev }
60549e08aacStbbdev 
60649e08aacStbbdev /** \page parallel_for_each_body_req Requirements on parallel_for_each body
60749e08aacStbbdev     Class \c Body implementing the concept of parallel_for_each body must define:
60849e08aacStbbdev     - \code
60949e08aacStbbdev         B::operator()(
61049e08aacStbbdev                 cv_item_type item,
61149e08aacStbbdev                 feeder<item_type>& feeder
61249e08aacStbbdev         ) const
61349e08aacStbbdev 
61449e08aacStbbdev         OR
61549e08aacStbbdev 
61649e08aacStbbdev         B::operator()( cv_item_type& item ) const
61749e08aacStbbdev       \endcode                                               Process item.
61849e08aacStbbdev                                                              May be invoked concurrently  for the same \c this but different \c item.
61949e08aacStbbdev 
62049e08aacStbbdev     - \code item_type( const item_type& ) \endcode
62149e08aacStbbdev                                                              Copy a work item.
62249e08aacStbbdev     - \code ~item_type() \endcode                            Destroy a work item
62349e08aacStbbdev **/
62449e08aacStbbdev 
62549e08aacStbbdev /** \name parallel_for_each
62649e08aacStbbdev     See also requirements on \ref parallel_for_each_body_req "parallel_for_each Body". **/
62749e08aacStbbdev //@{
62849e08aacStbbdev //! Parallel iteration over a range, with optional addition of more work.
62949e08aacStbbdev /** @ingroup algorithms */
63049e08aacStbbdev template<typename Iterator, typename Body>
631478de5b1Stbbdev     __TBB_requires(std::input_iterator<Iterator> && parallel_for_each_iterator_body<Body, Iterator>)
63249e08aacStbbdev void parallel_for_each(Iterator first, Iterator last, const Body& body) {
63349e08aacStbbdev     task_group_context context(PARALLEL_FOR_EACH);
63449e08aacStbbdev     run_parallel_for_each<Iterator, Body>(first, last, body, context);
63549e08aacStbbdev }
63649e08aacStbbdev 
63749e08aacStbbdev template<typename Range, typename Body>
638478de5b1Stbbdev     __TBB_requires(container_based_sequence<Range, std::input_iterator_tag> && parallel_for_each_range_body<Body, Range>)
63949e08aacStbbdev void parallel_for_each(Range& rng, const Body& body) {
64049e08aacStbbdev     parallel_for_each(std::begin(rng), std::end(rng), body);
64149e08aacStbbdev }
64249e08aacStbbdev 
64349e08aacStbbdev template<typename Range, typename Body>
644478de5b1Stbbdev     __TBB_requires(container_based_sequence<Range, std::input_iterator_tag> && parallel_for_each_range_body<Body, Range>)
64549e08aacStbbdev void parallel_for_each(const Range& rng, const Body& body) {
64649e08aacStbbdev     parallel_for_each(std::begin(rng), std::end(rng), body);
64749e08aacStbbdev }
64849e08aacStbbdev 
64949e08aacStbbdev //! Parallel iteration over a range, with optional addition of more work and user-supplied context
65049e08aacStbbdev /** @ingroup algorithms */
65149e08aacStbbdev template<typename Iterator, typename Body>
652478de5b1Stbbdev     __TBB_requires(std::input_iterator<Iterator> && parallel_for_each_iterator_body<Body, Iterator>)
65349e08aacStbbdev void parallel_for_each(Iterator first, Iterator last, const Body& body, task_group_context& context) {
65449e08aacStbbdev     run_parallel_for_each<Iterator, Body>(first, last, body, context);
65549e08aacStbbdev }
65649e08aacStbbdev 
65749e08aacStbbdev template<typename Range, typename Body>
658478de5b1Stbbdev     __TBB_requires(container_based_sequence<Range, std::input_iterator_tag> && parallel_for_each_range_body<Body, Range>)
65949e08aacStbbdev void parallel_for_each(Range& rng, const Body& body, task_group_context& context) {
66049e08aacStbbdev     parallel_for_each(std::begin(rng), std::end(rng), body, context);
66149e08aacStbbdev }
66249e08aacStbbdev 
66349e08aacStbbdev template<typename Range, typename Body>
664478de5b1Stbbdev     __TBB_requires(container_based_sequence<Range, std::input_iterator_tag> && parallel_for_each_range_body<Body, Range>)
66549e08aacStbbdev void parallel_for_each(const Range& rng, const Body& body, task_group_context& context) {
66649e08aacStbbdev     parallel_for_each(std::begin(rng), std::end(rng), body, context);
66749e08aacStbbdev }
66849e08aacStbbdev 
669b15aabb3Stbbdev } // namespace d2
67049e08aacStbbdev } // namespace detail
67149e08aacStbbdev //! @endcond
67249e08aacStbbdev //@}
67349e08aacStbbdev 
67449e08aacStbbdev inline namespace v1 {
675b15aabb3Stbbdev using detail::d2::parallel_for_each;
67649e08aacStbbdev using detail::d1::feeder;
67749e08aacStbbdev } // namespace v1
67849e08aacStbbdev 
67949e08aacStbbdev } // namespace tbb
68049e08aacStbbdev 
68149e08aacStbbdev #endif /* __TBB_parallel_for_each_H */
682