1 /*
2     Copyright (c) 2005-2021 Intel Corporation
3 
4     Licensed under the Apache License, Version 2.0 (the "License");
5     you may not use this file except in compliance with the License.
6     You may obtain a copy of the License at
7 
8         http://www.apache.org/licenses/LICENSE-2.0
9 
10     Unless required by applicable law or agreed to in writing, software
11     distributed under the License is distributed on an "AS IS" BASIS,
12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13     See the License for the specific language governing permissions and
14     limitations under the License.
15 */
16 
17 #ifndef __TBB_scalable_allocator_H
18 #define __TBB_scalable_allocator_H
19 
20 #ifdef __cplusplus
21 #include "oneapi/tbb/detail/_config.h"
22 #include "oneapi/tbb/detail/_utils.h"
23 #include <cstdlib>
24 #include <utility>
25 #else
26 #include <stddef.h> /* Need ptrdiff_t and size_t from here. */
27 #if !_MSC_VER
28 #include <stdint.h> /* Need intptr_t from here. */
29 #endif
30 #endif
31 
32 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
33 #include <memory_resource>
34 #endif
35 
36 #ifdef __cplusplus
37 extern "C" {
38 #endif /* __cplusplus */
39 
40 #if _MSC_VER
41     #define __TBB_EXPORTED_FUNC __cdecl
42 #else
43     #define __TBB_EXPORTED_FUNC
44 #endif
45 
46 /** The "malloc" analogue to allocate block of memory of size bytes.
47   * @ingroup memory_allocation */
48 void* __TBB_EXPORTED_FUNC scalable_malloc(size_t size);
49 
50 /** The "free" analogue to discard a previously allocated piece of memory.
51     @ingroup memory_allocation */
52 void   __TBB_EXPORTED_FUNC scalable_free(void* ptr);
53 
54 /** The "realloc" analogue complementing scalable_malloc.
55     @ingroup memory_allocation */
56 void* __TBB_EXPORTED_FUNC scalable_realloc(void* ptr, size_t size);
57 
58 /** The "calloc" analogue complementing scalable_malloc.
59     @ingroup memory_allocation */
60 void* __TBB_EXPORTED_FUNC scalable_calloc(size_t nobj, size_t size);
61 
62 /** The "posix_memalign" analogue.
63     @ingroup memory_allocation */
64 int __TBB_EXPORTED_FUNC scalable_posix_memalign(void** memptr, size_t alignment, size_t size);
65 
66 /** The "_aligned_malloc" analogue.
67     @ingroup memory_allocation */
68 void* __TBB_EXPORTED_FUNC scalable_aligned_malloc(size_t size, size_t alignment);
69 
70 /** The "_aligned_realloc" analogue.
71     @ingroup memory_allocation */
72 void* __TBB_EXPORTED_FUNC scalable_aligned_realloc(void* ptr, size_t size, size_t alignment);
73 
74 /** The "_aligned_free" analogue.
75     @ingroup memory_allocation */
76 void __TBB_EXPORTED_FUNC scalable_aligned_free(void* ptr);
77 
78 /** The analogue of _msize/malloc_size/malloc_usable_size.
79     Returns the usable size of a memory block previously allocated by scalable_*,
80     or 0 (zero) if ptr does not point to such a block.
81     @ingroup memory_allocation */
82 size_t __TBB_EXPORTED_FUNC scalable_msize(void* ptr);
83 
84 /* Results for scalable_allocation_* functions */
85 typedef enum {
86     TBBMALLOC_OK,
87     TBBMALLOC_INVALID_PARAM,
88     TBBMALLOC_UNSUPPORTED,
89     TBBMALLOC_NO_MEMORY,
90     TBBMALLOC_NO_EFFECT
91 } ScalableAllocationResult;
92 
93 /* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages.
94    scalable_allocation_mode call has priority over environment variable. */
95 typedef enum {
96     TBBMALLOC_USE_HUGE_PAGES,  /* value turns using huge pages on and off */
97     /* deprecated, kept for backward compatibility only */
98     USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES,
99     /* try to limit memory consumption value (Bytes), clean internal buffers
100        if limit is exceeded, but not prevents from requesting memory from OS */
101     TBBMALLOC_SET_SOFT_HEAP_LIMIT,
102     /* Lower bound for the size (Bytes), that is interpreted as huge
103      * and not released during regular cleanup operations. */
104     TBBMALLOC_SET_HUGE_SIZE_THRESHOLD
105 } AllocationModeParam;
106 
107 /** Set TBB allocator-specific allocation modes.
108     @ingroup memory_allocation */
109 int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value);
110 
111 typedef enum {
112     /* Clean internal allocator buffers for all threads.
113        Returns TBBMALLOC_NO_EFFECT if no buffers cleaned,
114        TBBMALLOC_OK if some memory released from buffers. */
115     TBBMALLOC_CLEAN_ALL_BUFFERS,
116     /* Clean internal allocator buffer for current thread only.
117        Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */
118     TBBMALLOC_CLEAN_THREAD_BUFFERS
119 } ScalableAllocationCmd;
120 
121 /** Call TBB allocator-specific commands.
122     @ingroup memory_allocation */
123 int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param);
124 
125 #ifdef __cplusplus
126 } /* extern "C" */
127 #endif /* __cplusplus */
128 
129 #ifdef __cplusplus
130 
131 //! The namespace rml contains components of low-level memory pool interface.
132 namespace rml {
133 class MemoryPool;
134 
135 typedef void *(*rawAllocType)(std::intptr_t pool_id, std::size_t &bytes);
136 // returns non-zero in case of error
137 typedef int   (*rawFreeType)(std::intptr_t pool_id, void* raw_ptr, std::size_t raw_bytes);
138 
139 struct MemPoolPolicy {
140     enum {
141         TBBMALLOC_POOL_VERSION = 1
142     };
143 
144     rawAllocType pAlloc;
145     rawFreeType  pFree;
146                  // granularity of pAlloc allocations. 0 means default used.
147     std::size_t  granularity;
148     int          version;
149                  // all memory consumed at 1st pAlloc call and never returned,
150                  // no more pAlloc calls after 1st
151     unsigned     fixedPool : 1,
152                  // memory consumed but returned only at pool termination
153                  keepAllMemory : 1,
154                  reserved : 30;
155 
156     MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_,
157                   std::size_t granularity_ = 0, bool fixedPool_ = false,
158                   bool keepAllMemory_ = false) :
159         pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION),
160         fixedPool(fixedPool_), keepAllMemory(keepAllMemory_),
161         reserved(0) {}
162 };
163 
164 // enums have same values as appropriate enums from ScalableAllocationResult
165 // TODO: use ScalableAllocationResult in pool_create directly
166 enum MemPoolError {
167     // pool created successfully
168     POOL_OK = TBBMALLOC_OK,
169     // invalid policy parameters found
170     INVALID_POLICY = TBBMALLOC_INVALID_PARAM,
171      // requested pool policy is not supported by allocator library
172     UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED,
173     // lack of memory during pool creation
174     NO_MEMORY = TBBMALLOC_NO_MEMORY,
175     // action takes no effect
176     NO_EFFECT = TBBMALLOC_NO_EFFECT
177 };
178 
179 MemPoolError pool_create_v1(std::intptr_t pool_id, const MemPoolPolicy *policy,
180                             rml::MemoryPool **pool);
181 
182 bool  pool_destroy(MemoryPool* memPool);
183 void *pool_malloc(MemoryPool* memPool, std::size_t size);
184 void *pool_realloc(MemoryPool* memPool, void *object, std::size_t size);
185 void *pool_aligned_malloc(MemoryPool* mPool, std::size_t size, std::size_t alignment);
186 void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, std::size_t size, std::size_t alignment);
187 bool  pool_reset(MemoryPool* memPool);
188 bool  pool_free(MemoryPool *memPool, void *object);
189 MemoryPool *pool_identify(void *object);
190 std::size_t pool_msize(MemoryPool *memPool, void *object);
191 
192 } // namespace rml
193 
194 namespace tbb {
195 namespace detail {
196 namespace d1 {
197 
198 // keep throw in a separate function to prevent code bloat
199 template<typename E>
200 void throw_exception(const E &e) {
201 #if TBB_USE_EXCEPTIONS
202     throw e;
203 #else
204     suppress_unused_warning(e);
205 #endif
206 }
207 
208 template<typename T>
209 class scalable_allocator {
210 public:
211     using value_type = T;
212     using propagate_on_container_move_assignment = std::true_type;
213 
214     //! Always defined for TBB containers
215     using is_always_equal = std::true_type;
216 
217     scalable_allocator() = default;
218     template<typename U> scalable_allocator(const scalable_allocator<U>&) noexcept {}
219 
220     //! Allocate space for n objects.
221     __TBB_nodiscard T* allocate(std::size_t n) {
222         T* p = static_cast<T*>(scalable_malloc(n * sizeof(value_type)));
223         if (!p) {
224             throw_exception(std::bad_alloc());
225         }
226         return p;
227     }
228 
229     //! Free previously allocated block of memory
230     void deallocate(T* p, std::size_t) {
231         scalable_free(p);
232     }
233 
234 #if TBB_ALLOCATOR_TRAITS_BROKEN
235     using pointer = value_type*;
236     using const_pointer = const value_type*;
237     using reference = value_type&;
238     using const_reference = const value_type&;
239     using difference_type = std::ptrdiff_t;
240     using size_type = std::size_t;
241     template<typename U> struct rebind {
242         using other = scalable_allocator<U>;
243     };
244     //! Largest value for which method allocate might succeed.
245     size_type max_size() const noexcept {
246         size_type absolutemax = static_cast<size_type>(-1) / sizeof (value_type);
247         return (absolutemax > 0 ? absolutemax : 1);
248     }
249     template<typename U, typename... Args>
250     void construct(U *p, Args&&... args)
251         { ::new((void *)p) U(std::forward<Args>(args)...); }
252     void destroy(pointer p) { p->~value_type(); }
253     pointer address(reference x) const { return &x; }
254     const_pointer address(const_reference x) const { return &x; }
255 #endif // TBB_ALLOCATOR_TRAITS_BROKEN
256 
257 };
258 
259 #if TBB_ALLOCATOR_TRAITS_BROKEN
260     template<>
261     class scalable_allocator<void> {
262     public:
263         using pointer = void*;
264         using const_pointer = const void*;
265         using value_type = void;
266         template<typename U> struct rebind {
267             using other = scalable_allocator<U>;
268         };
269     };
270 #endif
271 
272 template<typename T, typename U>
273 inline bool operator==(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return true; }
274 
275 #if !__TBB_CPP20_COMPARISONS_PRESENT
276 template<typename T, typename U>
277 inline bool operator!=(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return false; }
278 #endif
279 
280 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
281 
282 //! C++17 memory resource implementation for scalable allocator
283 //! ISO C++ Section 23.12.2
284 class scalable_resource_impl : public std::pmr::memory_resource {
285 private:
286     void* do_allocate(std::size_t bytes, std::size_t alignment) override {
287         void* p = scalable_aligned_malloc(bytes, alignment);
288         if (!p) {
289             throw_exception(std::bad_alloc());
290         }
291         return p;
292     }
293 
294     void do_deallocate(void* ptr, std::size_t /*bytes*/, std::size_t /*alignment*/) override {
295         scalable_free(ptr);
296     }
297 
298     //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any
299     //! other instance of this class
300     bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
301         return this == &other ||
302 #if __TBB_USE_OPTIONAL_RTTI
303             dynamic_cast<const scalable_resource_impl*>(&other) != nullptr;
304 #else
305             false;
306 #endif
307     }
308 };
309 
310 //! Global scalable allocator memory resource provider
311 inline std::pmr::memory_resource* scalable_memory_resource() noexcept {
312     static tbb::detail::d1::scalable_resource_impl scalable_res;
313     return &scalable_res;
314 }
315 
316 #endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT
317 
318 } // namespace d1
319 } // namespace detail
320 
321 inline namespace v1 {
322 using detail::d1::scalable_allocator;
323 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
324 using detail::d1::scalable_memory_resource;
325 #endif
326 } // namespace v1
327 
328 } // namespace tbb
329 
330 #endif /* __cplusplus */
331 
332 #endif /* __TBB_scalable_allocator_H */
333