149e08aacStbbdev /* 2e6e493f9SChristian Fersch Copyright (c) 2005-2023 Intel Corporation 349e08aacStbbdev 449e08aacStbbdev Licensed under the Apache License, Version 2.0 (the "License"); 549e08aacStbbdev you may not use this file except in compliance with the License. 649e08aacStbbdev You may obtain a copy of the License at 749e08aacStbbdev 849e08aacStbbdev http://www.apache.org/licenses/LICENSE-2.0 949e08aacStbbdev 1049e08aacStbbdev Unless required by applicable law or agreed to in writing, software 1149e08aacStbbdev distributed under the License is distributed on an "AS IS" BASIS, 1249e08aacStbbdev WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1349e08aacStbbdev See the License for the specific language governing permissions and 1449e08aacStbbdev limitations under the License. 1549e08aacStbbdev */ 1649e08aacStbbdev 1749e08aacStbbdev #ifndef __TBB_parallel_for_each_H 1849e08aacStbbdev #define __TBB_parallel_for_each_H 1949e08aacStbbdev 2049e08aacStbbdev #include "detail/_config.h" 2149e08aacStbbdev #include "detail/_namespace_injection.h" 2249e08aacStbbdev #include "detail/_exception.h" 2349e08aacStbbdev #include "detail/_task.h" 2449e08aacStbbdev #include "detail/_aligned_space.h" 2549e08aacStbbdev #include "detail/_small_object_pool.h" 26*a088cfa0SKonstantin Boyarinov #include "detail/_utils.h" 2749e08aacStbbdev 2849e08aacStbbdev #include "parallel_for.h" 2949e08aacStbbdev #include "task_group.h" // task_group_context 3049e08aacStbbdev 3149e08aacStbbdev #include <iterator> 3249e08aacStbbdev #include <type_traits> 3349e08aacStbbdev 3449e08aacStbbdev namespace tbb { 3549e08aacStbbdev namespace detail { 36478de5b1Stbbdev #if __TBB_CPP20_CONCEPTS_PRESENT 37478de5b1Stbbdev namespace d1 { 38478de5b1Stbbdev template <typename Item> 39478de5b1Stbbdev class feeder; 40478de5b1Stbbdev 41478de5b1Stbbdev } // namespace d1 42478de5b1Stbbdev inline namespace d0 { 43478de5b1Stbbdev 44478de5b1Stbbdev template <typename Body, typename ItemType, typename FeederItemType> 45*a088cfa0SKonstantin Boyarinov concept parallel_for_each_body = std::invocable<const std::remove_reference_t<Body>&, ItemType&&> || 46*a088cfa0SKonstantin Boyarinov std::invocable<const std::remove_reference_t<Body>&, ItemType&&, tbb::detail::d1::feeder<FeederItemType>&>; 47478de5b1Stbbdev 48478de5b1Stbbdev } // namespace d0 49478de5b1Stbbdev #endif // __TBB_CPP20_CONCEPTS_PRESENT 50b15aabb3Stbbdev namespace d2 { 5149e08aacStbbdev template<typename Body, typename Item> class feeder_impl; 52b15aabb3Stbbdev } // namespace d2 5349e08aacStbbdev 54b15aabb3Stbbdev namespace d1 { 5549e08aacStbbdev //! Class the user supplied algorithm body uses to add new tasks 5649e08aacStbbdev template<typename Item> 5749e08aacStbbdev class feeder { 5849e08aacStbbdev feeder() {} 5949e08aacStbbdev feeder(const feeder&) = delete; 6049e08aacStbbdev void operator=( const feeder&) = delete; 6149e08aacStbbdev 6249e08aacStbbdev virtual ~feeder () {} 6349e08aacStbbdev virtual void internal_add_copy(const Item& item) = 0; 6449e08aacStbbdev virtual void internal_add_move(Item&& item) = 0; 6549e08aacStbbdev 66478de5b1Stbbdev template<typename Body_, typename Item_> friend class d2::feeder_impl; 6749e08aacStbbdev public: 6849e08aacStbbdev //! Add a work item to a running parallel_for_each. 6949e08aacStbbdev void add(const Item& item) {internal_add_copy(item);} 7049e08aacStbbdev void add(Item&& item) {internal_add_move(std::move(item));} 7149e08aacStbbdev }; 7249e08aacStbbdev 73b15aabb3Stbbdev } // namespace d1 74b15aabb3Stbbdev 75b15aabb3Stbbdev namespace d2 { 76b15aabb3Stbbdev using namespace tbb::detail::d1; 7749e08aacStbbdev /** Selects one of the two possible forms of function call member operator. 7849e08aacStbbdev @ingroup algorithms **/ 79b15aabb3Stbbdev template<class Body> 80b15aabb3Stbbdev struct parallel_for_each_operator_selector { 8149e08aacStbbdev public: 8249e08aacStbbdev template<typename ItemArg, typename FeederArg> 83b15aabb3Stbbdev static auto call(const Body& body, ItemArg&& item, FeederArg*) 84*a088cfa0SKonstantin Boyarinov -> decltype(tbb::detail::invoke(body, std::forward<ItemArg>(item)), void()) { 8549e08aacStbbdev #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) 8649e08aacStbbdev // Suppression of Microsoft non-standard extension warnings 8749e08aacStbbdev #pragma warning (push) 8849e08aacStbbdev #pragma warning (disable: 4239) 8949e08aacStbbdev #endif 9049e08aacStbbdev 91*a088cfa0SKonstantin Boyarinov tbb::detail::invoke(body, std::forward<ItemArg>(item)); 9249e08aacStbbdev 9349e08aacStbbdev #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) 94e6e493f9SChristian Fersch #pragma warning (pop) 9549e08aacStbbdev #endif 9649e08aacStbbdev } 9749e08aacStbbdev 9849e08aacStbbdev template<typename ItemArg, typename FeederArg> 99b15aabb3Stbbdev static auto call(const Body& body, ItemArg&& item, FeederArg* feeder) 100*a088cfa0SKonstantin Boyarinov -> decltype(tbb::detail::invoke(body, std::forward<ItemArg>(item), *feeder), void()) { 10149e08aacStbbdev #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) 10249e08aacStbbdev // Suppression of Microsoft non-standard extension warnings 10349e08aacStbbdev #pragma warning (push) 10449e08aacStbbdev #pragma warning (disable: 4239) 10549e08aacStbbdev #endif 106b15aabb3Stbbdev __TBB_ASSERT(feeder, "Feeder was not created but should be"); 10749e08aacStbbdev 108*a088cfa0SKonstantin Boyarinov tbb::detail::invoke(body, std::forward<ItemArg>(item), *feeder); 10949e08aacStbbdev 11049e08aacStbbdev #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) 111e6e493f9SChristian Fersch #pragma warning (pop) 11249e08aacStbbdev #endif 11349e08aacStbbdev } 11449e08aacStbbdev }; 11549e08aacStbbdev 11649e08aacStbbdev template<typename Body, typename Item> 11749e08aacStbbdev struct feeder_item_task: public task { 11849e08aacStbbdev using feeder_type = feeder_impl<Body, Item>; 11949e08aacStbbdev 12049e08aacStbbdev template <typename ItemType> 12149e08aacStbbdev feeder_item_task(ItemType&& input_item, feeder_type& feeder, small_object_allocator& alloc) : 12249e08aacStbbdev item(std::forward<ItemType>(input_item)), 12349e08aacStbbdev my_feeder(feeder), 12449e08aacStbbdev my_allocator(alloc) 12549e08aacStbbdev {} 12649e08aacStbbdev 12749e08aacStbbdev void finalize(const execution_data& ed) { 12849e08aacStbbdev my_feeder.my_wait_context.release(); 12949e08aacStbbdev my_allocator.delete_object(this, ed); 13049e08aacStbbdev } 13149e08aacStbbdev 132b15aabb3Stbbdev //! Hack for resolve ambiguity between calls to the body with and without moving the stored copy 133b15aabb3Stbbdev //! Executing body with moving the copy should have higher priority 134b15aabb3Stbbdev using first_priority = int; 135b15aabb3Stbbdev using second_priority = double; 136b15aabb3Stbbdev 137b15aabb3Stbbdev template <typename BodyType, typename ItemType, typename FeederType> 138b15aabb3Stbbdev static auto call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, first_priority) 139b15aabb3Stbbdev -> decltype(parallel_for_each_operator_selector<Body>::call(call_body, std::move(call_item), &call_feeder), void()) 140b15aabb3Stbbdev { 141b15aabb3Stbbdev parallel_for_each_operator_selector<Body>::call(call_body, std::move(call_item), &call_feeder); 142b15aabb3Stbbdev } 143b15aabb3Stbbdev 144b15aabb3Stbbdev template <typename BodyType, typename ItemType, typename FeederType> 145b15aabb3Stbbdev static void call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, second_priority) { 146b15aabb3Stbbdev parallel_for_each_operator_selector<Body>::call(call_body, call_item, &call_feeder); 147b15aabb3Stbbdev } 148b15aabb3Stbbdev 14949e08aacStbbdev task* execute(execution_data& ed) override { 150b15aabb3Stbbdev call(my_feeder.my_body, item, my_feeder, first_priority{}); 15149e08aacStbbdev finalize(ed); 15249e08aacStbbdev return nullptr; 15349e08aacStbbdev } 15449e08aacStbbdev 15549e08aacStbbdev task* cancel(execution_data& ed) override { 15649e08aacStbbdev finalize(ed); 15749e08aacStbbdev return nullptr; 15849e08aacStbbdev } 15949e08aacStbbdev 16049e08aacStbbdev Item item; 16149e08aacStbbdev feeder_type& my_feeder; 16249e08aacStbbdev small_object_allocator my_allocator; 16349e08aacStbbdev }; // class feeder_item_task 16449e08aacStbbdev 16549e08aacStbbdev /** Implements new task adding procedure. 16649e08aacStbbdev @ingroup algorithms **/ 167b15aabb3Stbbdev template<typename Body, typename Item> 16849e08aacStbbdev class feeder_impl : public feeder<Item> { 16949e08aacStbbdev // Avoiding use of copy constructor in a virtual method if the type does not support it 17049e08aacStbbdev void internal_add_copy_impl(std::true_type, const Item& item) { 17149e08aacStbbdev using feeder_task = feeder_item_task<Body, Item>; 17249e08aacStbbdev small_object_allocator alloc; 17349e08aacStbbdev auto task = alloc.new_object<feeder_task>(item, *this, alloc); 17449e08aacStbbdev 17549e08aacStbbdev my_wait_context.reserve(); 17649e08aacStbbdev spawn(*task, my_execution_context); 17749e08aacStbbdev } 17849e08aacStbbdev 17949e08aacStbbdev void internal_add_copy_impl(std::false_type, const Item&) { 18049e08aacStbbdev __TBB_ASSERT(false, "Overloading for r-value reference doesn't work or it's not movable and not copyable object"); 18149e08aacStbbdev } 18249e08aacStbbdev 18349e08aacStbbdev void internal_add_copy(const Item& item) override { 18449e08aacStbbdev internal_add_copy_impl(typename std::is_copy_constructible<Item>::type(), item); 18549e08aacStbbdev } 18649e08aacStbbdev 18749e08aacStbbdev void internal_add_move(Item&& item) override { 18849e08aacStbbdev using feeder_task = feeder_item_task<Body, Item>; 18949e08aacStbbdev small_object_allocator alloc{}; 19049e08aacStbbdev auto task = alloc.new_object<feeder_task>(std::move(item), *this, alloc); 19149e08aacStbbdev 19249e08aacStbbdev my_wait_context.reserve(); 19349e08aacStbbdev spawn(*task, my_execution_context); 19449e08aacStbbdev } 19549e08aacStbbdev public: 196b15aabb3Stbbdev feeder_impl(const Body& body, wait_context& w_context, task_group_context &context) 197b15aabb3Stbbdev : my_body(body), 198b15aabb3Stbbdev my_wait_context(w_context) 19949e08aacStbbdev , my_execution_context(context) 20049e08aacStbbdev {} 20149e08aacStbbdev 20249e08aacStbbdev const Body& my_body; 203b15aabb3Stbbdev wait_context& my_wait_context; 20449e08aacStbbdev task_group_context& my_execution_context; 20549e08aacStbbdev }; // class feeder_impl 20649e08aacStbbdev 20749e08aacStbbdev /** Execute computation under one element of the range 20849e08aacStbbdev @ingroup algorithms **/ 20949e08aacStbbdev template<typename Iterator, typename Body, typename Item> 21049e08aacStbbdev struct for_each_iteration_task: public task { 21149e08aacStbbdev using feeder_type = feeder_impl<Body, Item>; 21249e08aacStbbdev 213b15aabb3Stbbdev for_each_iteration_task(Iterator input_item_ptr, const Body& body, feeder_impl<Body, Item>* feeder_ptr, wait_context& wait_context) : 214b15aabb3Stbbdev item_ptr(input_item_ptr), my_body(body), my_feeder_ptr(feeder_ptr), parent_wait_context(wait_context) 21549e08aacStbbdev {} 21649e08aacStbbdev 21749e08aacStbbdev void finalize() { 21849e08aacStbbdev parent_wait_context.release(); 21949e08aacStbbdev } 22049e08aacStbbdev 22149e08aacStbbdev task* execute(execution_data&) override { 222b15aabb3Stbbdev parallel_for_each_operator_selector<Body>::call(my_body, *item_ptr, my_feeder_ptr); 22349e08aacStbbdev finalize(); 22449e08aacStbbdev return nullptr; 22549e08aacStbbdev } 22649e08aacStbbdev 22749e08aacStbbdev task* cancel(execution_data&) override { 22849e08aacStbbdev finalize(); 22949e08aacStbbdev return nullptr; 23049e08aacStbbdev } 23149e08aacStbbdev 23249e08aacStbbdev Iterator item_ptr; 233b15aabb3Stbbdev const Body& my_body; 234b15aabb3Stbbdev feeder_impl<Body, Item>* my_feeder_ptr; 23549e08aacStbbdev wait_context& parent_wait_context; 236b15aabb3Stbbdev }; // class for_each_iteration_task 237b15aabb3Stbbdev 238b15aabb3Stbbdev // Helper to get the type of the iterator to the internal sequence of copies 239b15aabb3Stbbdev // If the element can be passed to the body as an rvalue - this iterator should be move_iterator 240b15aabb3Stbbdev template <typename Body, typename Item, typename = void> 241b15aabb3Stbbdev struct input_iteration_task_iterator_helper { 242b15aabb3Stbbdev // For input iterators we pass const lvalue reference to the body 243b15aabb3Stbbdev // It is prohibited to take non-constant lvalue references for input iterators 244b15aabb3Stbbdev using type = const Item*; 245b15aabb3Stbbdev }; 246b15aabb3Stbbdev 247b15aabb3Stbbdev template <typename Body, typename Item> 248b15aabb3Stbbdev struct input_iteration_task_iterator_helper<Body, Item, 249b15aabb3Stbbdev tbb::detail::void_t<decltype(parallel_for_each_operator_selector<Body>::call(std::declval<const Body&>(), 250b15aabb3Stbbdev std::declval<Item&&>(), 251b15aabb3Stbbdev std::declval<feeder_impl<Body, Item>*>()))>> 252b15aabb3Stbbdev { 253b15aabb3Stbbdev using type = std::move_iterator<Item*>; 254b15aabb3Stbbdev }; 25549e08aacStbbdev 25649e08aacStbbdev /** Split one block task to several(max_block_size) iteration tasks for input iterators 25749e08aacStbbdev @ingroup algorithms **/ 25849e08aacStbbdev template <typename Body, typename Item> 25949e08aacStbbdev struct input_block_handling_task : public task { 26049e08aacStbbdev static constexpr size_t max_block_size = 4; 26149e08aacStbbdev 26249e08aacStbbdev using feeder_type = feeder_impl<Body, Item>; 263b15aabb3Stbbdev using iteration_task_iterator_type = typename input_iteration_task_iterator_helper<Body, Item>::type; 264b15aabb3Stbbdev using iteration_task = for_each_iteration_task<iteration_task_iterator_type, Body, Item>; 26549e08aacStbbdev 266b15aabb3Stbbdev input_block_handling_task(wait_context& root_wait_context, task_group_context& e_context, 267b15aabb3Stbbdev const Body& body, feeder_impl<Body, Item>* feeder_ptr, small_object_allocator& alloc) 268b15aabb3Stbbdev :my_size(0), my_wait_context(0), my_root_wait_context(root_wait_context), 269b15aabb3Stbbdev my_execution_context(e_context), my_allocator(alloc) 27049e08aacStbbdev { 271b15aabb3Stbbdev auto item_it = block_iteration_space.begin(); 27249e08aacStbbdev for (auto* it = task_pool.begin(); it != task_pool.end(); ++it) { 273b15aabb3Stbbdev new (it) iteration_task(iteration_task_iterator_type(item_it++), body, feeder_ptr, my_wait_context); 27449e08aacStbbdev } 27549e08aacStbbdev } 27649e08aacStbbdev 27749e08aacStbbdev void finalize(const execution_data& ed) { 278b15aabb3Stbbdev my_root_wait_context.release(); 27949e08aacStbbdev my_allocator.delete_object(this, ed); 28049e08aacStbbdev } 28149e08aacStbbdev 28249e08aacStbbdev task* execute(execution_data& ed) override { 28349e08aacStbbdev __TBB_ASSERT( my_size > 0, "Negative size was passed to task"); 28449e08aacStbbdev for (std::size_t counter = 1; counter < my_size; ++counter) { 28549e08aacStbbdev my_wait_context.reserve(); 286b15aabb3Stbbdev spawn(*(task_pool.begin() + counter), my_execution_context); 28749e08aacStbbdev } 28849e08aacStbbdev my_wait_context.reserve(); 289b15aabb3Stbbdev execute_and_wait(*task_pool.begin(), my_execution_context, 290b15aabb3Stbbdev my_wait_context, my_execution_context); 29149e08aacStbbdev 29249e08aacStbbdev // deallocate current task after children execution 29349e08aacStbbdev finalize(ed); 29449e08aacStbbdev return nullptr; 29549e08aacStbbdev } 29649e08aacStbbdev 29749e08aacStbbdev task* cancel(execution_data& ed) override { 29849e08aacStbbdev finalize(ed); 29949e08aacStbbdev return nullptr; 30049e08aacStbbdev } 30149e08aacStbbdev 30249e08aacStbbdev ~input_block_handling_task() { 30349e08aacStbbdev for(std::size_t counter = 0; counter < max_block_size; ++counter) { 30449e08aacStbbdev (task_pool.begin() + counter)->~iteration_task(); 305aa247fa8SIlya Isaev if (counter < my_size) { 30649e08aacStbbdev (block_iteration_space.begin() + counter)->~Item(); 30749e08aacStbbdev } 30849e08aacStbbdev } 309aa247fa8SIlya Isaev } 31049e08aacStbbdev 31149e08aacStbbdev aligned_space<Item, max_block_size> block_iteration_space; 31249e08aacStbbdev aligned_space<iteration_task, max_block_size> task_pool; 31349e08aacStbbdev std::size_t my_size; 31449e08aacStbbdev wait_context my_wait_context; 315b15aabb3Stbbdev wait_context& my_root_wait_context; 316b15aabb3Stbbdev task_group_context& my_execution_context; 31749e08aacStbbdev small_object_allocator my_allocator; 318b15aabb3Stbbdev }; // class input_block_handling_task 31949e08aacStbbdev 32049e08aacStbbdev /** Split one block task to several(max_block_size) iteration tasks for forward iterators 32149e08aacStbbdev @ingroup algorithms **/ 32249e08aacStbbdev template <typename Iterator, typename Body, typename Item> 32349e08aacStbbdev struct forward_block_handling_task : public task { 32449e08aacStbbdev static constexpr size_t max_block_size = 4; 32549e08aacStbbdev 32649e08aacStbbdev using iteration_task = for_each_iteration_task<Iterator, Body, Item>; 32749e08aacStbbdev 328b15aabb3Stbbdev forward_block_handling_task(Iterator first, std::size_t size, 329b15aabb3Stbbdev wait_context& w_context, task_group_context& e_context, 330b15aabb3Stbbdev const Body& body, feeder_impl<Body, Item>* feeder_ptr, 331b15aabb3Stbbdev small_object_allocator& alloc) 332b15aabb3Stbbdev : my_size(size), my_wait_context(0), my_root_wait_context(w_context), 333b15aabb3Stbbdev my_execution_context(e_context), my_allocator(alloc) 33449e08aacStbbdev { 33549e08aacStbbdev auto* task_it = task_pool.begin(); 33649e08aacStbbdev for (std::size_t i = 0; i < size; i++) { 337b15aabb3Stbbdev new (task_it++) iteration_task(first, body, feeder_ptr, my_wait_context); 33849e08aacStbbdev ++first; 33949e08aacStbbdev } 34049e08aacStbbdev } 34149e08aacStbbdev 34249e08aacStbbdev void finalize(const execution_data& ed) { 343b15aabb3Stbbdev my_root_wait_context.release(); 34449e08aacStbbdev my_allocator.delete_object(this, ed); 34549e08aacStbbdev } 34649e08aacStbbdev 34749e08aacStbbdev task* execute(execution_data& ed) override { 34849e08aacStbbdev __TBB_ASSERT( my_size > 0, "Negative size was passed to task"); 34949e08aacStbbdev for(std::size_t counter = 1; counter < my_size; ++counter) { 35049e08aacStbbdev my_wait_context.reserve(); 351b15aabb3Stbbdev spawn(*(task_pool.begin() + counter), my_execution_context); 35249e08aacStbbdev } 35349e08aacStbbdev my_wait_context.reserve(); 354b15aabb3Stbbdev execute_and_wait(*task_pool.begin(), my_execution_context, 355b15aabb3Stbbdev my_wait_context, my_execution_context); 35649e08aacStbbdev 35749e08aacStbbdev // deallocate current task after children execution 35849e08aacStbbdev finalize(ed); 35949e08aacStbbdev return nullptr; 36049e08aacStbbdev } 36149e08aacStbbdev 36249e08aacStbbdev task* cancel(execution_data& ed) override { 36349e08aacStbbdev finalize(ed); 36449e08aacStbbdev return nullptr; 36549e08aacStbbdev } 36649e08aacStbbdev 36749e08aacStbbdev ~forward_block_handling_task() { 36849e08aacStbbdev for(std::size_t counter = 0; counter < my_size; ++counter) { 36949e08aacStbbdev (task_pool.begin() + counter)->~iteration_task(); 37049e08aacStbbdev } 37149e08aacStbbdev } 37249e08aacStbbdev 37349e08aacStbbdev aligned_space<iteration_task, max_block_size> task_pool; 37449e08aacStbbdev std::size_t my_size; 37549e08aacStbbdev wait_context my_wait_context; 376b15aabb3Stbbdev wait_context& my_root_wait_context; 377b15aabb3Stbbdev task_group_context& my_execution_context; 37849e08aacStbbdev small_object_allocator my_allocator; 37949e08aacStbbdev }; // class forward_block_handling_task 38049e08aacStbbdev 38149e08aacStbbdev /** Body for parallel_for algorithm. 38249e08aacStbbdev * Allows to redirect operations under random access iterators range to the parallel_for algorithm. 38349e08aacStbbdev @ingroup algorithms **/ 38449e08aacStbbdev template <typename Iterator, typename Body, typename Item> 38549e08aacStbbdev class parallel_for_body_wrapper { 38649e08aacStbbdev Iterator my_first; 387b15aabb3Stbbdev const Body& my_body; 388b15aabb3Stbbdev feeder_impl<Body, Item>* my_feeder_ptr; 38949e08aacStbbdev public: 390b15aabb3Stbbdev parallel_for_body_wrapper(Iterator first, const Body& body, feeder_impl<Body, Item>* feeder_ptr) 391b15aabb3Stbbdev : my_first(first), my_body(body), my_feeder_ptr(feeder_ptr) {} 39249e08aacStbbdev 39349e08aacStbbdev void operator()(tbb::blocked_range<std::size_t> range) const { 39449e08aacStbbdev #if __INTEL_COMPILER 39549e08aacStbbdev #pragma ivdep 39649e08aacStbbdev #endif 39749e08aacStbbdev for (std::size_t count = range.begin(); count != range.end(); count++) { 398b15aabb3Stbbdev parallel_for_each_operator_selector<Body>::call(my_body, *(my_first + count), 399b15aabb3Stbbdev my_feeder_ptr); 40049e08aacStbbdev } 40149e08aacStbbdev } 40249e08aacStbbdev }; // class parallel_for_body_wrapper 40349e08aacStbbdev 40449e08aacStbbdev 40549e08aacStbbdev /** Helper for getting iterators tag including inherited custom tags 40649e08aacStbbdev @ingroup algorithms */ 40749e08aacStbbdev template<typename It> 40849e08aacStbbdev using tag = typename std::iterator_traits<It>::iterator_category; 40949e08aacStbbdev 41049e08aacStbbdev template<typename It> 41149e08aacStbbdev using iterator_tag_dispatch = typename 41249e08aacStbbdev std::conditional< 41349e08aacStbbdev std::is_base_of<std::random_access_iterator_tag, tag<It>>::value, 41449e08aacStbbdev std::random_access_iterator_tag, 41549e08aacStbbdev typename std::conditional< 41649e08aacStbbdev std::is_base_of<std::forward_iterator_tag, tag<It>>::value, 41749e08aacStbbdev std::forward_iterator_tag, 41849e08aacStbbdev std::input_iterator_tag 41949e08aacStbbdev >::type 42049e08aacStbbdev >::type; 42149e08aacStbbdev 422b15aabb3Stbbdev template <typename Body, typename Iterator, typename Item> 423*a088cfa0SKonstantin Boyarinov using feeder_is_required = tbb::detail::void_t<decltype(tbb::detail::invoke(std::declval<const Body>(), 424*a088cfa0SKonstantin Boyarinov std::declval<typename std::iterator_traits<Iterator>::reference>(), 425b15aabb3Stbbdev std::declval<feeder<Item>&>()))>; 426b15aabb3Stbbdev 427b15aabb3Stbbdev // Creates feeder object only if the body can accept it 428b15aabb3Stbbdev template <typename Iterator, typename Body, typename Item, typename = void> 429b15aabb3Stbbdev struct feeder_holder { 430b15aabb3Stbbdev feeder_holder( wait_context&, task_group_context&, const Body& ) {} 431b15aabb3Stbbdev 432b15aabb3Stbbdev feeder_impl<Body, Item>* feeder_ptr() { return nullptr; } 433b15aabb3Stbbdev }; // class feeder_holder 434b15aabb3Stbbdev 435b15aabb3Stbbdev template <typename Iterator, typename Body, typename Item> 436b15aabb3Stbbdev class feeder_holder<Iterator, Body, Item, feeder_is_required<Body, Iterator, Item>> { 437b15aabb3Stbbdev public: 438b15aabb3Stbbdev feeder_holder( wait_context& w_context, task_group_context& context, const Body& body ) 439b15aabb3Stbbdev : my_feeder(body, w_context, context) {} 440b15aabb3Stbbdev 441b15aabb3Stbbdev feeder_impl<Body, Item>* feeder_ptr() { return &my_feeder; } 442b15aabb3Stbbdev private: 443b15aabb3Stbbdev feeder_impl<Body, Item> my_feeder; 444b15aabb3Stbbdev }; // class feeder_holder 445b15aabb3Stbbdev 446b15aabb3Stbbdev template <typename Iterator, typename Body, typename Item> 447b15aabb3Stbbdev class for_each_root_task_base : public task { 448b15aabb3Stbbdev public: 449b15aabb3Stbbdev for_each_root_task_base(Iterator first, Iterator last, const Body& body, wait_context& w_context, task_group_context& e_context) 450b15aabb3Stbbdev : my_first(first), my_last(last), my_wait_context(w_context), my_execution_context(e_context), 451b15aabb3Stbbdev my_body(body), my_feeder_holder(my_wait_context, my_execution_context, my_body) 452b15aabb3Stbbdev { 453b15aabb3Stbbdev my_wait_context.reserve(); 454b15aabb3Stbbdev } 455b15aabb3Stbbdev private: 456b15aabb3Stbbdev task* cancel(execution_data&) override { 457b15aabb3Stbbdev this->my_wait_context.release(); 458b15aabb3Stbbdev return nullptr; 459b15aabb3Stbbdev } 460b15aabb3Stbbdev protected: 461b15aabb3Stbbdev Iterator my_first; 462b15aabb3Stbbdev Iterator my_last; 463b15aabb3Stbbdev wait_context& my_wait_context; 464b15aabb3Stbbdev task_group_context& my_execution_context; 465b15aabb3Stbbdev const Body& my_body; 466b15aabb3Stbbdev feeder_holder<Iterator, Body, Item> my_feeder_holder; 467b15aabb3Stbbdev }; // class for_each_root_task_base 468b15aabb3Stbbdev 46949e08aacStbbdev /** parallel_for_each algorithm root task - most generic version 47049e08aacStbbdev * Splits input range to blocks 47149e08aacStbbdev @ingroup algorithms **/ 47249e08aacStbbdev template <typename Iterator, typename Body, typename Item, typename IteratorTag = iterator_tag_dispatch<Iterator>> 473b15aabb3Stbbdev class for_each_root_task : public for_each_root_task_base<Iterator, Body, Item> 47449e08aacStbbdev { 475b15aabb3Stbbdev using base_type = for_each_root_task_base<Iterator, Body, Item>; 476b15aabb3Stbbdev public: 477b15aabb3Stbbdev using base_type::base_type; 47849e08aacStbbdev private: 47949e08aacStbbdev task* execute(execution_data& ed) override { 48049e08aacStbbdev using block_handling_type = input_block_handling_task<Body, Item>; 48149e08aacStbbdev 482b15aabb3Stbbdev if (this->my_first == this->my_last) { 483b15aabb3Stbbdev this->my_wait_context.release(); 48449e08aacStbbdev return nullptr; 48549e08aacStbbdev } 48649e08aacStbbdev 487b15aabb3Stbbdev this->my_wait_context.reserve(); 48849e08aacStbbdev small_object_allocator alloc{}; 489b15aabb3Stbbdev auto block_handling_task = alloc.new_object<block_handling_type>(ed, this->my_wait_context, this->my_execution_context, 490b15aabb3Stbbdev this->my_body, this->my_feeder_holder.feeder_ptr(), 491b15aabb3Stbbdev alloc); 49249e08aacStbbdev 49349e08aacStbbdev auto* block_iterator = block_handling_task->block_iteration_space.begin(); 494b15aabb3Stbbdev for (; !(this->my_first == this->my_last) && block_handling_task->my_size < block_handling_type::max_block_size; ++this->my_first) { 49549e08aacStbbdev // Move semantics are automatically used when supported by the iterator 496b15aabb3Stbbdev new (block_iterator++) Item(*this->my_first); 49749e08aacStbbdev ++block_handling_task->my_size; 49849e08aacStbbdev } 49949e08aacStbbdev 50049e08aacStbbdev // Do not access this after spawn to avoid races 501b15aabb3Stbbdev spawn(*this, this->my_execution_context); 50249e08aacStbbdev return block_handling_task; 50349e08aacStbbdev } 50449e08aacStbbdev }; // class for_each_root_task - most generic implementation 50549e08aacStbbdev 50649e08aacStbbdev /** parallel_for_each algorithm root task - forward iterator based specialization 50749e08aacStbbdev * Splits input range to blocks 50849e08aacStbbdev @ingroup algorithms **/ 50949e08aacStbbdev template <typename Iterator, typename Body, typename Item> 510b15aabb3Stbbdev class for_each_root_task<Iterator, Body, Item, std::forward_iterator_tag> 511b15aabb3Stbbdev : public for_each_root_task_base<Iterator, Body, Item> 51249e08aacStbbdev { 513b15aabb3Stbbdev using base_type = for_each_root_task_base<Iterator, Body, Item>; 514b15aabb3Stbbdev public: 515b15aabb3Stbbdev using base_type::base_type; 51649e08aacStbbdev private: 51749e08aacStbbdev task* execute(execution_data& ed) override { 51849e08aacStbbdev using block_handling_type = forward_block_handling_task<Iterator, Body, Item>; 519b15aabb3Stbbdev if (this->my_first == this->my_last) { 520b15aabb3Stbbdev this->my_wait_context.release(); 52149e08aacStbbdev return nullptr; 52249e08aacStbbdev } 52349e08aacStbbdev 52449e08aacStbbdev std::size_t block_size{0}; 525b15aabb3Stbbdev Iterator first_block_element = this->my_first; 526b15aabb3Stbbdev for (; !(this->my_first == this->my_last) && block_size < block_handling_type::max_block_size; ++this->my_first) { 52749e08aacStbbdev ++block_size; 52849e08aacStbbdev } 52949e08aacStbbdev 530b15aabb3Stbbdev this->my_wait_context.reserve(); 53149e08aacStbbdev small_object_allocator alloc{}; 532b15aabb3Stbbdev auto block_handling_task = alloc.new_object<block_handling_type>(ed, first_block_element, block_size, 533b15aabb3Stbbdev this->my_wait_context, this->my_execution_context, 534b15aabb3Stbbdev this->my_body, this->my_feeder_holder.feeder_ptr(), alloc); 53549e08aacStbbdev 53649e08aacStbbdev // Do not access this after spawn to avoid races 537b15aabb3Stbbdev spawn(*this, this->my_execution_context); 53849e08aacStbbdev return block_handling_task; 53949e08aacStbbdev } 54049e08aacStbbdev }; // class for_each_root_task - forward iterator based specialization 54149e08aacStbbdev 54249e08aacStbbdev /** parallel_for_each algorithm root task - random access iterator based specialization 54349e08aacStbbdev * Splits input range to blocks 54449e08aacStbbdev @ingroup algorithms **/ 54549e08aacStbbdev template <typename Iterator, typename Body, typename Item> 546b15aabb3Stbbdev class for_each_root_task<Iterator, Body, Item, std::random_access_iterator_tag> 547b15aabb3Stbbdev : public for_each_root_task_base<Iterator, Body, Item> 54849e08aacStbbdev { 549b15aabb3Stbbdev using base_type = for_each_root_task_base<Iterator, Body, Item>; 550b15aabb3Stbbdev public: 551b15aabb3Stbbdev using base_type::base_type; 55249e08aacStbbdev private: 55349e08aacStbbdev task* execute(execution_data&) override { 55449e08aacStbbdev tbb::parallel_for( 555b15aabb3Stbbdev tbb::blocked_range<std::size_t>(0, std::distance(this->my_first, this->my_last)), 556b15aabb3Stbbdev parallel_for_body_wrapper<Iterator, Body, Item>(this->my_first, this->my_body, this->my_feeder_holder.feeder_ptr()) 557b15aabb3Stbbdev , this->my_execution_context 55849e08aacStbbdev ); 55949e08aacStbbdev 560b15aabb3Stbbdev this->my_wait_context.release(); 56149e08aacStbbdev return nullptr; 56249e08aacStbbdev } 56349e08aacStbbdev }; // class for_each_root_task - random access iterator based specialization 56449e08aacStbbdev 56549e08aacStbbdev /** Helper for getting item type. If item type can be deduced from feeder - got it from feeder, 56649e08aacStbbdev if feeder is generic - got item type from range. 56749e08aacStbbdev @ingroup algorithms */ 56849e08aacStbbdev template<typename Body, typename Item, typename FeederArg> 56949e08aacStbbdev auto feeder_argument_parser(void (Body::*)(Item, feeder<FeederArg>&) const) -> FeederArg; 57049e08aacStbbdev 57149e08aacStbbdev template<typename Body, typename> 57249e08aacStbbdev decltype(feeder_argument_parser<Body>(&Body::operator())) get_item_type_impl(int); // for (T, feeder<T>) 57349e08aacStbbdev template<typename Body, typename Item> Item get_item_type_impl(...); // stub 57449e08aacStbbdev 57549e08aacStbbdev template <typename Body, typename Item> 57649e08aacStbbdev using get_item_type = decltype(get_item_type_impl<Body, Item>(0)); 57749e08aacStbbdev 578478de5b1Stbbdev #if __TBB_CPP20_CONCEPTS_PRESENT 579478de5b1Stbbdev template <typename Body, typename ItemType> 580478de5b1Stbbdev using feeder_item_type = std::remove_cvref_t<get_item_type<Body, ItemType>>; 581478de5b1Stbbdev 582478de5b1Stbbdev template <typename Body, typename Iterator> 583478de5b1Stbbdev concept parallel_for_each_iterator_body = 584478de5b1Stbbdev parallel_for_each_body<Body, iterator_reference_type<Iterator>, feeder_item_type<Body, iterator_reference_type<Iterator>>>; 585478de5b1Stbbdev 586478de5b1Stbbdev template <typename Body, typename Range> 587478de5b1Stbbdev concept parallel_for_each_range_body = 588478de5b1Stbbdev parallel_for_each_body<Body, range_reference_type<Range>, feeder_item_type<Body, range_reference_type<Range>>>; 589478de5b1Stbbdev #endif 590478de5b1Stbbdev 59149e08aacStbbdev /** Implements parallel iteration over a range. 59249e08aacStbbdev @ingroup algorithms */ 59349e08aacStbbdev template<typename Iterator, typename Body> 59449e08aacStbbdev void run_parallel_for_each( Iterator first, Iterator last, const Body& body, task_group_context& context) 59549e08aacStbbdev { 59649e08aacStbbdev if (!(first == last)) { 597b15aabb3Stbbdev using ItemType = get_item_type<Body, typename std::iterator_traits<Iterator>::value_type>; 598b15aabb3Stbbdev wait_context w_context(0); 59949e08aacStbbdev 600b15aabb3Stbbdev for_each_root_task<Iterator, Body, ItemType> root_task(first, last, body, w_context, context); 60149e08aacStbbdev 602b15aabb3Stbbdev execute_and_wait(root_task, context, w_context, context); 60349e08aacStbbdev } 60449e08aacStbbdev } 60549e08aacStbbdev 60649e08aacStbbdev /** \page parallel_for_each_body_req Requirements on parallel_for_each body 60749e08aacStbbdev Class \c Body implementing the concept of parallel_for_each body must define: 60849e08aacStbbdev - \code 60949e08aacStbbdev B::operator()( 61049e08aacStbbdev cv_item_type item, 61149e08aacStbbdev feeder<item_type>& feeder 61249e08aacStbbdev ) const 61349e08aacStbbdev 61449e08aacStbbdev OR 61549e08aacStbbdev 61649e08aacStbbdev B::operator()( cv_item_type& item ) const 61749e08aacStbbdev \endcode Process item. 61849e08aacStbbdev May be invoked concurrently for the same \c this but different \c item. 61949e08aacStbbdev 62049e08aacStbbdev - \code item_type( const item_type& ) \endcode 62149e08aacStbbdev Copy a work item. 62249e08aacStbbdev - \code ~item_type() \endcode Destroy a work item 62349e08aacStbbdev **/ 62449e08aacStbbdev 62549e08aacStbbdev /** \name parallel_for_each 62649e08aacStbbdev See also requirements on \ref parallel_for_each_body_req "parallel_for_each Body". **/ 62749e08aacStbbdev //@{ 62849e08aacStbbdev //! Parallel iteration over a range, with optional addition of more work. 62949e08aacStbbdev /** @ingroup algorithms */ 63049e08aacStbbdev template<typename Iterator, typename Body> 631478de5b1Stbbdev __TBB_requires(std::input_iterator<Iterator> && parallel_for_each_iterator_body<Body, Iterator>) 63249e08aacStbbdev void parallel_for_each(Iterator first, Iterator last, const Body& body) { 63349e08aacStbbdev task_group_context context(PARALLEL_FOR_EACH); 63449e08aacStbbdev run_parallel_for_each<Iterator, Body>(first, last, body, context); 63549e08aacStbbdev } 63649e08aacStbbdev 63749e08aacStbbdev template<typename Range, typename Body> 638478de5b1Stbbdev __TBB_requires(container_based_sequence<Range, std::input_iterator_tag> && parallel_for_each_range_body<Body, Range>) 63949e08aacStbbdev void parallel_for_each(Range& rng, const Body& body) { 64049e08aacStbbdev parallel_for_each(std::begin(rng), std::end(rng), body); 64149e08aacStbbdev } 64249e08aacStbbdev 64349e08aacStbbdev template<typename Range, typename Body> 644478de5b1Stbbdev __TBB_requires(container_based_sequence<Range, std::input_iterator_tag> && parallel_for_each_range_body<Body, Range>) 64549e08aacStbbdev void parallel_for_each(const Range& rng, const Body& body) { 64649e08aacStbbdev parallel_for_each(std::begin(rng), std::end(rng), body); 64749e08aacStbbdev } 64849e08aacStbbdev 64949e08aacStbbdev //! Parallel iteration over a range, with optional addition of more work and user-supplied context 65049e08aacStbbdev /** @ingroup algorithms */ 65149e08aacStbbdev template<typename Iterator, typename Body> 652478de5b1Stbbdev __TBB_requires(std::input_iterator<Iterator> && parallel_for_each_iterator_body<Body, Iterator>) 65349e08aacStbbdev void parallel_for_each(Iterator first, Iterator last, const Body& body, task_group_context& context) { 65449e08aacStbbdev run_parallel_for_each<Iterator, Body>(first, last, body, context); 65549e08aacStbbdev } 65649e08aacStbbdev 65749e08aacStbbdev template<typename Range, typename Body> 658478de5b1Stbbdev __TBB_requires(container_based_sequence<Range, std::input_iterator_tag> && parallel_for_each_range_body<Body, Range>) 65949e08aacStbbdev void parallel_for_each(Range& rng, const Body& body, task_group_context& context) { 66049e08aacStbbdev parallel_for_each(std::begin(rng), std::end(rng), body, context); 66149e08aacStbbdev } 66249e08aacStbbdev 66349e08aacStbbdev template<typename Range, typename Body> 664478de5b1Stbbdev __TBB_requires(container_based_sequence<Range, std::input_iterator_tag> && parallel_for_each_range_body<Body, Range>) 66549e08aacStbbdev void parallel_for_each(const Range& rng, const Body& body, task_group_context& context) { 66649e08aacStbbdev parallel_for_each(std::begin(rng), std::end(rng), body, context); 66749e08aacStbbdev } 66849e08aacStbbdev 669b15aabb3Stbbdev } // namespace d2 67049e08aacStbbdev } // namespace detail 67149e08aacStbbdev //! @endcond 67249e08aacStbbdev //@} 67349e08aacStbbdev 67449e08aacStbbdev inline namespace v1 { 675b15aabb3Stbbdev using detail::d2::parallel_for_each; 67649e08aacStbbdev using detail::d1::feeder; 67749e08aacStbbdev } // namespace v1 67849e08aacStbbdev 67949e08aacStbbdev } // namespace tbb 68049e08aacStbbdev 68149e08aacStbbdev #endif /* __TBB_parallel_for_each_H */ 682