1 /*
2 Copyright (c) 2005-2023 Intel Corporation
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 #ifndef __TBB_scalable_allocator_H
18 #define __TBB_scalable_allocator_H
19
20 #ifdef __cplusplus
21 #include "oneapi/tbb/detail/_config.h"
22 #include "oneapi/tbb/detail/_utils.h"
23 #include "oneapi/tbb/detail/_namespace_injection.h"
24 #include <cstdlib>
25 #include <utility>
26 #include <new> /* std::bad_alloc() */
27 #else
28 #include "oneapi/tbb/detail/_export.h"
29 #include <stddef.h> /* Need ptrdiff_t and size_t from here. */
30 #if !defined(_MSC_VER) || defined(__clang__)
31 #include <stdint.h> /* Need intptr_t from here. */
32 #endif
33 #endif
34
35 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
36 #include <memory_resource>
37 #endif
38
39 #ifdef __cplusplus
40 extern "C" {
41 #endif /* __cplusplus */
42
43 #if _MSC_VER
44 #define __TBB_EXPORTED_FUNC __cdecl
45 #else
46 #define __TBB_EXPORTED_FUNC
47 #endif
48
49 /** The "malloc" analogue to allocate block of memory of size bytes.
50 * @ingroup memory_allocation */
51 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_malloc(size_t size);
52
53 /** The "free" analogue to discard a previously allocated piece of memory.
54 @ingroup memory_allocation */
55 TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_free(void* ptr);
56
57 /** The "realloc" analogue complementing scalable_malloc.
58 @ingroup memory_allocation */
59 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_realloc(void* ptr, size_t size);
60
61 /** The "calloc" analogue complementing scalable_malloc.
62 @ingroup memory_allocation */
63 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_calloc(size_t nobj, size_t size);
64
65 /** The "posix_memalign" analogue.
66 @ingroup memory_allocation */
67 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_posix_memalign(void** memptr, size_t alignment, size_t size);
68
69 /** The "_aligned_malloc" analogue.
70 @ingroup memory_allocation */
71 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_malloc(size_t size, size_t alignment);
72
73 /** The "_aligned_realloc" analogue.
74 @ingroup memory_allocation */
75 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_realloc(void* ptr, size_t size, size_t alignment);
76
77 /** The "_aligned_free" analogue.
78 @ingroup memory_allocation */
79 TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_aligned_free(void* ptr);
80
81 /** The analogue of _msize/malloc_size/malloc_usable_size.
82 Returns the usable size of a memory block previously allocated by scalable_*,
83 or 0 (zero) if ptr does not point to such a block.
84 @ingroup memory_allocation */
85 TBBMALLOC_EXPORT size_t __TBB_EXPORTED_FUNC scalable_msize(void* ptr);
86
87 /* Results for scalable_allocation_* functions */
88 typedef enum {
89 TBBMALLOC_OK,
90 TBBMALLOC_INVALID_PARAM,
91 TBBMALLOC_UNSUPPORTED,
92 TBBMALLOC_NO_MEMORY,
93 TBBMALLOC_NO_EFFECT
94 } ScalableAllocationResult;
95
96 /* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages.
97 scalable_allocation_mode call has priority over environment variable. */
98 typedef enum {
99 TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */
100 /* deprecated, kept for backward compatibility only */
101 USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES,
102 /* try to limit memory consumption value (Bytes), clean internal buffers
103 if limit is exceeded, but not prevents from requesting memory from OS */
104 TBBMALLOC_SET_SOFT_HEAP_LIMIT,
105 /* Lower bound for the size (Bytes), that is interpreted as huge
106 * and not released during regular cleanup operations. */
107 TBBMALLOC_SET_HUGE_SIZE_THRESHOLD
108 } AllocationModeParam;
109
110 /** Set TBB allocator-specific allocation modes.
111 @ingroup memory_allocation */
112 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value);
113
114 typedef enum {
115 /* Clean internal allocator buffers for all threads.
116 Returns TBBMALLOC_NO_EFFECT if no buffers cleaned,
117 TBBMALLOC_OK if some memory released from buffers. */
118 TBBMALLOC_CLEAN_ALL_BUFFERS,
119 /* Clean internal allocator buffer for current thread only.
120 Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */
121 TBBMALLOC_CLEAN_THREAD_BUFFERS
122 } ScalableAllocationCmd;
123
124 /** Call TBB allocator-specific commands.
125 @ingroup memory_allocation */
126 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param);
127
128 #ifdef __cplusplus
129 } /* extern "C" */
130 #endif /* __cplusplus */
131
132 #ifdef __cplusplus
133
134 //! The namespace rml contains components of low-level memory pool interface.
135 namespace rml {
136 class MemoryPool;
137
138 typedef void *(*rawAllocType)(std::intptr_t pool_id, std::size_t &bytes);
139 // returns non-zero in case of error
140 typedef int (*rawFreeType)(std::intptr_t pool_id, void* raw_ptr, std::size_t raw_bytes);
141
142 struct MemPoolPolicy {
143 enum {
144 TBBMALLOC_POOL_VERSION = 1
145 };
146
147 rawAllocType pAlloc;
148 rawFreeType pFree;
149 // granularity of pAlloc allocations. 0 means default used.
150 std::size_t granularity;
151 int version;
152 // all memory consumed at 1st pAlloc call and never returned,
153 // no more pAlloc calls after 1st
154 unsigned fixedPool : 1,
155 // memory consumed but returned only at pool termination
156 keepAllMemory : 1,
157 reserved : 30;
158
159 MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_,
160 std::size_t granularity_ = 0, bool fixedPool_ = false,
161 bool keepAllMemory_ = false) :
pAllocMemPoolPolicy162 pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION),
163 fixedPool(fixedPool_), keepAllMemory(keepAllMemory_),
164 reserved(0) {}
165 };
166
167 // enums have same values as appropriate enums from ScalableAllocationResult
168 // TODO: use ScalableAllocationResult in pool_create directly
169 enum MemPoolError {
170 // pool created successfully
171 POOL_OK = TBBMALLOC_OK,
172 // invalid policy parameters found
173 INVALID_POLICY = TBBMALLOC_INVALID_PARAM,
174 // requested pool policy is not supported by allocator library
175 UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED,
176 // lack of memory during pool creation
177 NO_MEMORY = TBBMALLOC_NO_MEMORY,
178 // action takes no effect
179 NO_EFFECT = TBBMALLOC_NO_EFFECT
180 };
181
182 TBBMALLOC_EXPORT MemPoolError pool_create_v1(std::intptr_t pool_id, const MemPoolPolicy *policy,
183 rml::MemoryPool **pool);
184
185 TBBMALLOC_EXPORT bool pool_destroy(MemoryPool* memPool);
186 TBBMALLOC_EXPORT void *pool_malloc(MemoryPool* memPool, std::size_t size);
187 TBBMALLOC_EXPORT void *pool_realloc(MemoryPool* memPool, void *object, std::size_t size);
188 TBBMALLOC_EXPORT void *pool_aligned_malloc(MemoryPool* mPool, std::size_t size, std::size_t alignment);
189 TBBMALLOC_EXPORT void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, std::size_t size, std::size_t alignment);
190 TBBMALLOC_EXPORT bool pool_reset(MemoryPool* memPool);
191 TBBMALLOC_EXPORT bool pool_free(MemoryPool *memPool, void *object);
192 TBBMALLOC_EXPORT MemoryPool *pool_identify(void *object);
193 TBBMALLOC_EXPORT std::size_t pool_msize(MemoryPool *memPool, void *object);
194
195 } // namespace rml
196
197 namespace tbb {
198 namespace detail {
199 namespace d1 {
200
201 // keep throw in a separate function to prevent code bloat
202 template<typename E>
throw_exception(const E & e)203 void throw_exception(const E &e) {
204 #if TBB_USE_EXCEPTIONS
205 throw e;
206 #else
207 suppress_unused_warning(e);
208 #endif
209 }
210
211 template<typename T>
212 class scalable_allocator {
213 public:
214 using value_type = T;
215 using propagate_on_container_move_assignment = std::true_type;
216
217 //! Always defined for TBB containers
218 using is_always_equal = std::true_type;
219
220 scalable_allocator() = default;
scalable_allocator(const scalable_allocator<U> &)221 template<typename U> scalable_allocator(const scalable_allocator<U>&) noexcept {}
222
223 //! Allocate space for n objects.
allocate(std::size_t n)224 __TBB_nodiscard T* allocate(std::size_t n) {
225 T* p = static_cast<T*>(scalable_malloc(n * sizeof(value_type)));
226 if (!p) {
227 throw_exception(std::bad_alloc());
228 }
229 return p;
230 }
231
232 //! Free previously allocated block of memory
deallocate(T * p,std::size_t)233 void deallocate(T* p, std::size_t) {
234 scalable_free(p);
235 }
236
237 #if TBB_ALLOCATOR_TRAITS_BROKEN
238 using pointer = value_type*;
239 using const_pointer = const value_type*;
240 using reference = value_type&;
241 using const_reference = const value_type&;
242 using difference_type = std::ptrdiff_t;
243 using size_type = std::size_t;
244 template<typename U> struct rebind {
245 using other = scalable_allocator<U>;
246 };
247 //! Largest value for which method allocate might succeed.
max_size()248 size_type max_size() const noexcept {
249 size_type absolutemax = static_cast<size_type>(-1) / sizeof (value_type);
250 return (absolutemax > 0 ? absolutemax : 1);
251 }
252 template<typename U, typename... Args>
construct(U * p,Args &&...args)253 void construct(U *p, Args&&... args)
254 { ::new((void *)p) U(std::forward<Args>(args)...); }
destroy(pointer p)255 void destroy(pointer p) { p->~value_type(); }
address(reference x)256 pointer address(reference x) const { return &x; }
address(const_reference x)257 const_pointer address(const_reference x) const { return &x; }
258 #endif // TBB_ALLOCATOR_TRAITS_BROKEN
259
260 };
261
262 #if TBB_ALLOCATOR_TRAITS_BROKEN
263 template<>
264 class scalable_allocator<void> {
265 public:
266 using pointer = void*;
267 using const_pointer = const void*;
268 using value_type = void;
269 template<typename U> struct rebind {
270 using other = scalable_allocator<U>;
271 };
272 };
273 #endif
274
275 template<typename T, typename U>
276 inline bool operator==(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return true; }
277
278 #if !__TBB_CPP20_COMPARISONS_PRESENT
279 template<typename T, typename U>
280 inline bool operator!=(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return false; }
281 #endif
282
283 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
284
285 //! C++17 memory resource implementation for scalable allocator
286 //! ISO C++ Section 23.12.2
287 class scalable_resource_impl : public std::pmr::memory_resource {
288 private:
do_allocate(std::size_t bytes,std::size_t alignment)289 void* do_allocate(std::size_t bytes, std::size_t alignment) override {
290 void* p = scalable_aligned_malloc(bytes, alignment);
291 if (!p) {
292 throw_exception(std::bad_alloc());
293 }
294 return p;
295 }
296
do_deallocate(void * ptr,std::size_t,std::size_t)297 void do_deallocate(void* ptr, std::size_t /*bytes*/, std::size_t /*alignment*/) override {
298 scalable_free(ptr);
299 }
300
301 //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any
302 //! other instance of this class
do_is_equal(const std::pmr::memory_resource & other)303 bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
304 return this == &other ||
305 #if __TBB_USE_OPTIONAL_RTTI
306 dynamic_cast<const scalable_resource_impl*>(&other) != nullptr;
307 #else
308 false;
309 #endif
310 }
311 };
312
313 //! Global scalable allocator memory resource provider
scalable_memory_resource()314 inline std::pmr::memory_resource* scalable_memory_resource() noexcept {
315 static tbb::detail::d1::scalable_resource_impl scalable_res;
316 return &scalable_res;
317 }
318
319 #endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT
320
321 } // namespace d1
322 } // namespace detail
323
324 inline namespace v1 {
325 using detail::d1::scalable_allocator;
326 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
327 using detail::d1::scalable_memory_resource;
328 #endif
329 } // namespace v1
330
331 } // namespace tbb
332
333 #endif /* __cplusplus */
334
335 #endif /* __TBB_scalable_allocator_H */
336