1 /* 2 Copyright (c) 2005-2023 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #ifndef __TBB_scalable_allocator_H 18 #define __TBB_scalable_allocator_H 19 20 #ifdef __cplusplus 21 #include "oneapi/tbb/detail/_config.h" 22 #include "oneapi/tbb/detail/_utils.h" 23 #include "oneapi/tbb/detail/_namespace_injection.h" 24 #include <cstdlib> 25 #include <utility> 26 #include <new> /* std::bad_alloc() */ 27 #else 28 #include "oneapi/tbb/detail/_export.h" 29 #include <stddef.h> /* Need ptrdiff_t and size_t from here. */ 30 #if !defined(_MSC_VER) || defined(__clang__) 31 #include <stdint.h> /* Need intptr_t from here. */ 32 #endif 33 #endif 34 35 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT 36 #include <memory_resource> 37 #endif 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif /* __cplusplus */ 42 43 #if _MSC_VER 44 #define __TBB_EXPORTED_FUNC __cdecl 45 #else 46 #define __TBB_EXPORTED_FUNC 47 #endif 48 49 /** The "malloc" analogue to allocate block of memory of size bytes. 50 * @ingroup memory_allocation */ 51 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_malloc(size_t size); 52 53 /** The "free" analogue to discard a previously allocated piece of memory. 54 @ingroup memory_allocation */ 55 TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_free(void* ptr); 56 57 /** The "realloc" analogue complementing scalable_malloc. 58 @ingroup memory_allocation */ 59 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_realloc(void* ptr, size_t size); 60 61 /** The "calloc" analogue complementing scalable_malloc. 62 @ingroup memory_allocation */ 63 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_calloc(size_t nobj, size_t size); 64 65 /** The "posix_memalign" analogue. 66 @ingroup memory_allocation */ 67 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_posix_memalign(void** memptr, size_t alignment, size_t size); 68 69 /** The "_aligned_malloc" analogue. 70 @ingroup memory_allocation */ 71 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_malloc(size_t size, size_t alignment); 72 73 /** The "_aligned_realloc" analogue. 74 @ingroup memory_allocation */ 75 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_realloc(void* ptr, size_t size, size_t alignment); 76 77 /** The "_aligned_free" analogue. 78 @ingroup memory_allocation */ 79 TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_aligned_free(void* ptr); 80 81 /** The analogue of _msize/malloc_size/malloc_usable_size. 82 Returns the usable size of a memory block previously allocated by scalable_*, 83 or 0 (zero) if ptr does not point to such a block. 84 @ingroup memory_allocation */ 85 TBBMALLOC_EXPORT size_t __TBB_EXPORTED_FUNC scalable_msize(void* ptr); 86 87 /* Results for scalable_allocation_* functions */ 88 typedef enum { 89 TBBMALLOC_OK, 90 TBBMALLOC_INVALID_PARAM, 91 TBBMALLOC_UNSUPPORTED, 92 TBBMALLOC_NO_MEMORY, 93 TBBMALLOC_NO_EFFECT 94 } ScalableAllocationResult; 95 96 /* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages. 97 scalable_allocation_mode call has priority over environment variable. */ 98 typedef enum { 99 TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ 100 /* deprecated, kept for backward compatibility only */ 101 USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, 102 /* try to limit memory consumption value (Bytes), clean internal buffers 103 if limit is exceeded, but not prevents from requesting memory from OS */ 104 TBBMALLOC_SET_SOFT_HEAP_LIMIT, 105 /* Lower bound for the size (Bytes), that is interpreted as huge 106 * and not released during regular cleanup operations. */ 107 TBBMALLOC_SET_HUGE_SIZE_THRESHOLD 108 } AllocationModeParam; 109 110 /** Set TBB allocator-specific allocation modes. 111 @ingroup memory_allocation */ 112 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value); 113 114 typedef enum { 115 /* Clean internal allocator buffers for all threads. 116 Returns TBBMALLOC_NO_EFFECT if no buffers cleaned, 117 TBBMALLOC_OK if some memory released from buffers. */ 118 TBBMALLOC_CLEAN_ALL_BUFFERS, 119 /* Clean internal allocator buffer for current thread only. 120 Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */ 121 TBBMALLOC_CLEAN_THREAD_BUFFERS 122 } ScalableAllocationCmd; 123 124 /** Call TBB allocator-specific commands. 125 @ingroup memory_allocation */ 126 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param); 127 128 #ifdef __cplusplus 129 } /* extern "C" */ 130 #endif /* __cplusplus */ 131 132 #ifdef __cplusplus 133 134 //! The namespace rml contains components of low-level memory pool interface. 135 namespace rml { 136 class MemoryPool; 137 138 typedef void *(*rawAllocType)(std::intptr_t pool_id, std::size_t &bytes); 139 // returns non-zero in case of error 140 typedef int (*rawFreeType)(std::intptr_t pool_id, void* raw_ptr, std::size_t raw_bytes); 141 142 struct MemPoolPolicy { 143 enum { 144 TBBMALLOC_POOL_VERSION = 1 145 }; 146 147 rawAllocType pAlloc; 148 rawFreeType pFree; 149 // granularity of pAlloc allocations. 0 means default used. 150 std::size_t granularity; 151 int version; 152 // all memory consumed at 1st pAlloc call and never returned, 153 // no more pAlloc calls after 1st 154 unsigned fixedPool : 1, 155 // memory consumed but returned only at pool termination 156 keepAllMemory : 1, 157 reserved : 30; 158 159 MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_, 160 std::size_t granularity_ = 0, bool fixedPool_ = false, 161 bool keepAllMemory_ = false) : 162 pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION), 163 fixedPool(fixedPool_), keepAllMemory(keepAllMemory_), 164 reserved(0) {} 165 }; 166 167 // enums have same values as appropriate enums from ScalableAllocationResult 168 // TODO: use ScalableAllocationResult in pool_create directly 169 enum MemPoolError { 170 // pool created successfully 171 POOL_OK = TBBMALLOC_OK, 172 // invalid policy parameters found 173 INVALID_POLICY = TBBMALLOC_INVALID_PARAM, 174 // requested pool policy is not supported by allocator library 175 UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED, 176 // lack of memory during pool creation 177 NO_MEMORY = TBBMALLOC_NO_MEMORY, 178 // action takes no effect 179 NO_EFFECT = TBBMALLOC_NO_EFFECT 180 }; 181 182 TBBMALLOC_EXPORT MemPoolError pool_create_v1(std::intptr_t pool_id, const MemPoolPolicy *policy, 183 rml::MemoryPool **pool); 184 185 TBBMALLOC_EXPORT bool pool_destroy(MemoryPool* memPool); 186 TBBMALLOC_EXPORT void *pool_malloc(MemoryPool* memPool, std::size_t size); 187 TBBMALLOC_EXPORT void *pool_realloc(MemoryPool* memPool, void *object, std::size_t size); 188 TBBMALLOC_EXPORT void *pool_aligned_malloc(MemoryPool* mPool, std::size_t size, std::size_t alignment); 189 TBBMALLOC_EXPORT void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, std::size_t size, std::size_t alignment); 190 TBBMALLOC_EXPORT bool pool_reset(MemoryPool* memPool); 191 TBBMALLOC_EXPORT bool pool_free(MemoryPool *memPool, void *object); 192 TBBMALLOC_EXPORT MemoryPool *pool_identify(void *object); 193 TBBMALLOC_EXPORT std::size_t pool_msize(MemoryPool *memPool, void *object); 194 195 } // namespace rml 196 197 namespace tbb { 198 namespace detail { 199 namespace d1 { 200 201 // keep throw in a separate function to prevent code bloat 202 template<typename E> 203 void throw_exception(const E &e) { 204 #if TBB_USE_EXCEPTIONS 205 throw e; 206 #else 207 suppress_unused_warning(e); 208 #endif 209 } 210 211 template<typename T> 212 class scalable_allocator { 213 public: 214 using value_type = T; 215 using propagate_on_container_move_assignment = std::true_type; 216 217 //! Always defined for TBB containers 218 using is_always_equal = std::true_type; 219 220 scalable_allocator() = default; 221 template<typename U> scalable_allocator(const scalable_allocator<U>&) noexcept {} 222 223 //! Allocate space for n objects. 224 __TBB_nodiscard T* allocate(std::size_t n) { 225 T* p = static_cast<T*>(scalable_malloc(n * sizeof(value_type))); 226 if (!p) { 227 throw_exception(std::bad_alloc()); 228 } 229 return p; 230 } 231 232 //! Free previously allocated block of memory 233 void deallocate(T* p, std::size_t) { 234 scalable_free(p); 235 } 236 237 #if TBB_ALLOCATOR_TRAITS_BROKEN 238 using pointer = value_type*; 239 using const_pointer = const value_type*; 240 using reference = value_type&; 241 using const_reference = const value_type&; 242 using difference_type = std::ptrdiff_t; 243 using size_type = std::size_t; 244 template<typename U> struct rebind { 245 using other = scalable_allocator<U>; 246 }; 247 //! Largest value for which method allocate might succeed. 248 size_type max_size() const noexcept { 249 size_type absolutemax = static_cast<size_type>(-1) / sizeof (value_type); 250 return (absolutemax > 0 ? absolutemax : 1); 251 } 252 template<typename U, typename... Args> 253 void construct(U *p, Args&&... args) 254 { ::new((void *)p) U(std::forward<Args>(args)...); } 255 void destroy(pointer p) { p->~value_type(); } 256 pointer address(reference x) const { return &x; } 257 const_pointer address(const_reference x) const { return &x; } 258 #endif // TBB_ALLOCATOR_TRAITS_BROKEN 259 260 }; 261 262 #if TBB_ALLOCATOR_TRAITS_BROKEN 263 template<> 264 class scalable_allocator<void> { 265 public: 266 using pointer = void*; 267 using const_pointer = const void*; 268 using value_type = void; 269 template<typename U> struct rebind { 270 using other = scalable_allocator<U>; 271 }; 272 }; 273 #endif 274 275 template<typename T, typename U> 276 inline bool operator==(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return true; } 277 278 #if !__TBB_CPP20_COMPARISONS_PRESENT 279 template<typename T, typename U> 280 inline bool operator!=(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return false; } 281 #endif 282 283 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT 284 285 //! C++17 memory resource implementation for scalable allocator 286 //! ISO C++ Section 23.12.2 287 class scalable_resource_impl : public std::pmr::memory_resource { 288 private: 289 void* do_allocate(std::size_t bytes, std::size_t alignment) override { 290 void* p = scalable_aligned_malloc(bytes, alignment); 291 if (!p) { 292 throw_exception(std::bad_alloc()); 293 } 294 return p; 295 } 296 297 void do_deallocate(void* ptr, std::size_t /*bytes*/, std::size_t /*alignment*/) override { 298 scalable_free(ptr); 299 } 300 301 //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any 302 //! other instance of this class 303 bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { 304 return this == &other || 305 #if __TBB_USE_OPTIONAL_RTTI 306 dynamic_cast<const scalable_resource_impl*>(&other) != nullptr; 307 #else 308 false; 309 #endif 310 } 311 }; 312 313 //! Global scalable allocator memory resource provider 314 inline std::pmr::memory_resource* scalable_memory_resource() noexcept { 315 static tbb::detail::d1::scalable_resource_impl scalable_res; 316 return &scalable_res; 317 } 318 319 #endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT 320 321 } // namespace d1 322 } // namespace detail 323 324 inline namespace v1 { 325 using detail::d1::scalable_allocator; 326 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT 327 using detail::d1::scalable_memory_resource; 328 #endif 329 } // namespace v1 330 331 } // namespace tbb 332 333 #endif /* __cplusplus */ 334 335 #endif /* __TBB_scalable_allocator_H */ 336