1 /*
2     Copyright (c) 2005-2020 Intel Corporation
3 
4     Licensed under the Apache License, Version 2.0 (the "License");
5     you may not use this file except in compliance with the License.
6     You may obtain a copy of the License at
7 
8         http://www.apache.org/licenses/LICENSE-2.0
9 
10     Unless required by applicable law or agreed to in writing, software
11     distributed under the License is distributed on an "AS IS" BASIS,
12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13     See the License for the specific language governing permissions and
14     limitations under the License.
15 */
16 
17 #ifndef __TBB_cache_aligned_allocator_H
18 #define __TBB_cache_aligned_allocator_H
19 
20 #include "tbb/detail/_utils.h"
21 #include <cstdlib>
22 #include <utility>
23 
24 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
25 #include <memory_resource>
26 #endif
27 
28 namespace tbb {
29 namespace detail {
30 
31 namespace r1 {
32 void*       __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size);
33 void        __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p);
34 std::size_t __TBB_EXPORTED_FUNC cache_line_size();
35 }
36 
37 namespace d1 {
38 
39 template<typename T>
40 class cache_aligned_allocator {
41 public:
42     using value_type = T;
43     using propagate_on_container_move_assignment = std::true_type;
44 
45     //! Always defined for TBB containers (supported since C++17 for std containers)
46     using is_always_equal = std::true_type;
47 
48     cache_aligned_allocator() = default;
49     template<typename U> cache_aligned_allocator(const cache_aligned_allocator<U>&) noexcept {}
50 
51     //! Allocate space for n objects, starting on a cache/sector line.
52     T* allocate(std::size_t n) {
53         return static_cast<T*>(r1::cache_aligned_allocate(n * sizeof(value_type)));
54     }
55 
56     //! Free block of memory that starts on a cache line
57     void deallocate(T* p, std::size_t) {
58         r1::cache_aligned_deallocate(p);
59     }
60 
61     //! Largest value for which method allocate might succeed.
62     std::size_t max_size() const noexcept {
63         return (~std::size_t(0) - r1::cache_line_size()) / sizeof(value_type);
64     }
65 
66 #if TBB_ALLOCATOR_TRAITS_BROKEN
67     using pointer = value_type*;
68     using const_pointer = const value_type*;
69     using reference = value_type&;
70     using const_reference = const value_type&;
71     using difference_type = std::ptrdiff_t;
72     using size_type = std::size_t;
73     template<typename U> struct rebind {
74         using other = cache_aligned_allocator<U>;
75     };
76     template<typename U, typename... Args>
77     void construct(U *p, Args&&... args)
78         { ::new (p) U(std::forward<Args>(args)...); }
79     void destroy(pointer p) { p->~value_type(); }
80     pointer address(reference x) const { return &x; }
81     const_pointer address(const_reference x) const { return &x; }
82 #endif // TBB_ALLOCATOR_TRAITS_BROKEN
83 };
84 
85 #if TBB_ALLOCATOR_TRAITS_BROKEN
86     template<>
87     class cache_aligned_allocator<void> {
88     public:
89         using pointer = void*;
90         using const_pointer = const void*;
91         using value_type = void;
92         template<typename U> struct rebind {
93             using other = cache_aligned_allocator<U>;
94         };
95     };
96 #endif
97 
98 template<typename T, typename U>
99 bool operator==(const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>&) noexcept { return true; }
100 
101 template<typename T, typename U>
102 bool operator!=(const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>&) noexcept { return false; }
103 
104 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
105 
106 //! C++17 memory resource wrapper to ensure cache line size alignment
107 class cache_aligned_resource : public std::pmr::memory_resource {
108 public:
109     cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {}
110     explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {}
111 
112     std::pmr::memory_resource* upstream_resource() const {
113         return m_upstream;
114     }
115 
116 private:
117     //! We don't know what memory resource set. Use padding to guarantee alignment
118     void* do_allocate(std::size_t bytes, std::size_t alignment) override {
119         // TODO: make it common with tbb_allocator.cpp
120         std::size_t cache_line_alignment = correct_alignment(alignment);
121         std::size_t space = correct_size(bytes) + cache_line_alignment;
122         std::uintptr_t base = reinterpret_cast<std::uintptr_t>(m_upstream->allocate(space));
123         __TBB_ASSERT(base != 0, "Upstream resource returned NULL.");
124 
125         // Round up to the next cache line (align the base address)
126         std::uintptr_t result = (base + cache_line_alignment) & ~(cache_line_alignment - 1);
127         __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), "Can`t store a base pointer to the header");
128         __TBB_ASSERT(space - (result - base) >= bytes, "Not enough space for the storage");
129 
130         // Record where block actually starts.
131         (reinterpret_cast<std::uintptr_t*>(result))[-1] = base;
132         return reinterpret_cast<void*>(result);
133     }
134 
135     void do_deallocate(void* ptr, std::size_t bytes, std::size_t alignment) override {
136         if (ptr) {
137             // Recover where block actually starts
138             std::uintptr_t base = (reinterpret_cast<std::uintptr_t*>(ptr))[-1];
139             m_upstream->deallocate(reinterpret_cast<void*>(base), correct_size(bytes) + correct_alignment(alignment));
140         }
141     }
142 
143     bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
144         if (this == &other) { return true; }
145 #if __TBB_USE_OPTIONAL_RTTI
146         const cache_aligned_resource* other_res = dynamic_cast<const cache_aligned_resource*>(&other);
147         return other_res && (upstream_resource() == other_res->upstream_resource());
148 #else
149         return false;
150 #endif
151     }
152 
153     std::size_t correct_alignment(std::size_t alignment) {
154         __TBB_ASSERT(tbb::detail::is_power_of_two(alignment), "Alignment is not a power of 2");
155 #if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT
156         std::size_t cache_line_size = std::hardware_destructive_interference_size;
157 #else
158         std::size_t cache_line_size = r1::cache_line_size();
159 #endif
160         return alignment < cache_line_size ? cache_line_size : alignment;
161     }
162 
163     std::size_t correct_size(std::size_t bytes) {
164         // To handle the case, when small size requested. There could be not
165         // enough space to store the original pointer.
166         return bytes < sizeof(std::uintptr_t) ? sizeof(std::uintptr_t) : bytes;
167     }
168 
169     std::pmr::memory_resource* m_upstream;
170 };
171 
172 #endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT
173 
174 } // namespace d1
175 } // namespace detail
176 
177 inline namespace v1 {
178 using detail::d1::cache_aligned_allocator;
179 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
180 using detail::d1::cache_aligned_resource;
181 #endif
182 } // namespace v1
183 } // namespace tbb
184 
185 #endif /* __TBB_cache_aligned_allocator_H */
186 
187