1 /* 2 Copyright (c) 2005-2021 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #ifndef __TBB_scalable_allocator_H 18 #define __TBB_scalable_allocator_H 19 20 #ifdef __cplusplus 21 #include "oneapi/tbb/detail/_config.h" 22 #include "oneapi/tbb/detail/_utils.h" 23 #include <cstdlib> 24 #include <utility> 25 #include <new> /* std::bad_alloc() */ 26 #else 27 #include "oneapi/tbb/detail/_export.h" 28 #include <stddef.h> /* Need ptrdiff_t and size_t from here. */ 29 #if !defined(_MSC_VER) || defined(__clang__) 30 #include <stdint.h> /* Need intptr_t from here. */ 31 #endif 32 #endif 33 34 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT 35 #include <memory_resource> 36 #endif 37 38 #ifdef __cplusplus 39 extern "C" { 40 #endif /* __cplusplus */ 41 42 #if _MSC_VER 43 #define __TBB_EXPORTED_FUNC __cdecl 44 #else 45 #define __TBB_EXPORTED_FUNC 46 #endif 47 48 /** The "malloc" analogue to allocate block of memory of size bytes. 49 * @ingroup memory_allocation */ 50 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_malloc(size_t size); 51 52 /** The "free" analogue to discard a previously allocated piece of memory. 53 @ingroup memory_allocation */ 54 TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_free(void* ptr); 55 56 /** The "realloc" analogue complementing scalable_malloc. 57 @ingroup memory_allocation */ 58 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_realloc(void* ptr, size_t size); 59 60 /** The "calloc" analogue complementing scalable_malloc. 61 @ingroup memory_allocation */ 62 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_calloc(size_t nobj, size_t size); 63 64 /** The "posix_memalign" analogue. 65 @ingroup memory_allocation */ 66 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_posix_memalign(void** memptr, size_t alignment, size_t size); 67 68 /** The "_aligned_malloc" analogue. 69 @ingroup memory_allocation */ 70 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_malloc(size_t size, size_t alignment); 71 72 /** The "_aligned_realloc" analogue. 73 @ingroup memory_allocation */ 74 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_realloc(void* ptr, size_t size, size_t alignment); 75 76 /** The "_aligned_free" analogue. 77 @ingroup memory_allocation */ 78 TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_aligned_free(void* ptr); 79 80 /** The analogue of _msize/malloc_size/malloc_usable_size. 81 Returns the usable size of a memory block previously allocated by scalable_*, 82 or 0 (zero) if ptr does not point to such a block. 83 @ingroup memory_allocation */ 84 TBBMALLOC_EXPORT size_t __TBB_EXPORTED_FUNC scalable_msize(void* ptr); 85 86 /* Results for scalable_allocation_* functions */ 87 typedef enum { 88 TBBMALLOC_OK, 89 TBBMALLOC_INVALID_PARAM, 90 TBBMALLOC_UNSUPPORTED, 91 TBBMALLOC_NO_MEMORY, 92 TBBMALLOC_NO_EFFECT 93 } ScalableAllocationResult; 94 95 /* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages. 96 scalable_allocation_mode call has priority over environment variable. */ 97 typedef enum { 98 TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ 99 /* deprecated, kept for backward compatibility only */ 100 USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, 101 /* try to limit memory consumption value (Bytes), clean internal buffers 102 if limit is exceeded, but not prevents from requesting memory from OS */ 103 TBBMALLOC_SET_SOFT_HEAP_LIMIT, 104 /* Lower bound for the size (Bytes), that is interpreted as huge 105 * and not released during regular cleanup operations. */ 106 TBBMALLOC_SET_HUGE_SIZE_THRESHOLD 107 } AllocationModeParam; 108 109 /** Set TBB allocator-specific allocation modes. 110 @ingroup memory_allocation */ 111 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value); 112 113 typedef enum { 114 /* Clean internal allocator buffers for all threads. 115 Returns TBBMALLOC_NO_EFFECT if no buffers cleaned, 116 TBBMALLOC_OK if some memory released from buffers. */ 117 TBBMALLOC_CLEAN_ALL_BUFFERS, 118 /* Clean internal allocator buffer for current thread only. 119 Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */ 120 TBBMALLOC_CLEAN_THREAD_BUFFERS 121 } ScalableAllocationCmd; 122 123 /** Call TBB allocator-specific commands. 124 @ingroup memory_allocation */ 125 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param); 126 127 #ifdef __cplusplus 128 } /* extern "C" */ 129 #endif /* __cplusplus */ 130 131 #ifdef __cplusplus 132 133 //! The namespace rml contains components of low-level memory pool interface. 134 namespace rml { 135 class MemoryPool; 136 137 typedef void *(*rawAllocType)(std::intptr_t pool_id, std::size_t &bytes); 138 // returns non-zero in case of error 139 typedef int (*rawFreeType)(std::intptr_t pool_id, void* raw_ptr, std::size_t raw_bytes); 140 141 struct MemPoolPolicy { 142 enum { 143 TBBMALLOC_POOL_VERSION = 1 144 }; 145 146 rawAllocType pAlloc; 147 rawFreeType pFree; 148 // granularity of pAlloc allocations. 0 means default used. 149 std::size_t granularity; 150 int version; 151 // all memory consumed at 1st pAlloc call and never returned, 152 // no more pAlloc calls after 1st 153 unsigned fixedPool : 1, 154 // memory consumed but returned only at pool termination 155 keepAllMemory : 1, 156 reserved : 30; 157 158 MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_, 159 std::size_t granularity_ = 0, bool fixedPool_ = false, 160 bool keepAllMemory_ = false) : 161 pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION), 162 fixedPool(fixedPool_), keepAllMemory(keepAllMemory_), 163 reserved(0) {} 164 }; 165 166 // enums have same values as appropriate enums from ScalableAllocationResult 167 // TODO: use ScalableAllocationResult in pool_create directly 168 enum MemPoolError { 169 // pool created successfully 170 POOL_OK = TBBMALLOC_OK, 171 // invalid policy parameters found 172 INVALID_POLICY = TBBMALLOC_INVALID_PARAM, 173 // requested pool policy is not supported by allocator library 174 UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED, 175 // lack of memory during pool creation 176 NO_MEMORY = TBBMALLOC_NO_MEMORY, 177 // action takes no effect 178 NO_EFFECT = TBBMALLOC_NO_EFFECT 179 }; 180 181 TBBMALLOC_EXPORT MemPoolError pool_create_v1(std::intptr_t pool_id, const MemPoolPolicy *policy, 182 rml::MemoryPool **pool); 183 184 TBBMALLOC_EXPORT bool pool_destroy(MemoryPool* memPool); 185 TBBMALLOC_EXPORT void *pool_malloc(MemoryPool* memPool, std::size_t size); 186 TBBMALLOC_EXPORT void *pool_realloc(MemoryPool* memPool, void *object, std::size_t size); 187 TBBMALLOC_EXPORT void *pool_aligned_malloc(MemoryPool* mPool, std::size_t size, std::size_t alignment); 188 TBBMALLOC_EXPORT void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, std::size_t size, std::size_t alignment); 189 TBBMALLOC_EXPORT bool pool_reset(MemoryPool* memPool); 190 TBBMALLOC_EXPORT bool pool_free(MemoryPool *memPool, void *object); 191 TBBMALLOC_EXPORT MemoryPool *pool_identify(void *object); 192 TBBMALLOC_EXPORT std::size_t pool_msize(MemoryPool *memPool, void *object); 193 194 } // namespace rml 195 196 namespace tbb { 197 namespace detail { 198 namespace d1 { 199 200 // keep throw in a separate function to prevent code bloat 201 template<typename E> 202 void throw_exception(const E &e) { 203 #if TBB_USE_EXCEPTIONS 204 throw e; 205 #else 206 suppress_unused_warning(e); 207 #endif 208 } 209 210 template<typename T> 211 class scalable_allocator { 212 public: 213 using value_type = T; 214 using propagate_on_container_move_assignment = std::true_type; 215 216 //! Always defined for TBB containers 217 using is_always_equal = std::true_type; 218 219 scalable_allocator() = default; 220 template<typename U> scalable_allocator(const scalable_allocator<U>&) noexcept {} 221 222 //! Allocate space for n objects. 223 __TBB_nodiscard T* allocate(std::size_t n) { 224 T* p = static_cast<T*>(scalable_malloc(n * sizeof(value_type))); 225 if (!p) { 226 throw_exception(std::bad_alloc()); 227 } 228 return p; 229 } 230 231 //! Free previously allocated block of memory 232 void deallocate(T* p, std::size_t) { 233 scalable_free(p); 234 } 235 236 #if TBB_ALLOCATOR_TRAITS_BROKEN 237 using pointer = value_type*; 238 using const_pointer = const value_type*; 239 using reference = value_type&; 240 using const_reference = const value_type&; 241 using difference_type = std::ptrdiff_t; 242 using size_type = std::size_t; 243 template<typename U> struct rebind { 244 using other = scalable_allocator<U>; 245 }; 246 //! Largest value for which method allocate might succeed. 247 size_type max_size() const noexcept { 248 size_type absolutemax = static_cast<size_type>(-1) / sizeof (value_type); 249 return (absolutemax > 0 ? absolutemax : 1); 250 } 251 template<typename U, typename... Args> 252 void construct(U *p, Args&&... args) 253 { ::new((void *)p) U(std::forward<Args>(args)...); } 254 void destroy(pointer p) { p->~value_type(); } 255 pointer address(reference x) const { return &x; } 256 const_pointer address(const_reference x) const { return &x; } 257 #endif // TBB_ALLOCATOR_TRAITS_BROKEN 258 259 }; 260 261 #if TBB_ALLOCATOR_TRAITS_BROKEN 262 template<> 263 class scalable_allocator<void> { 264 public: 265 using pointer = void*; 266 using const_pointer = const void*; 267 using value_type = void; 268 template<typename U> struct rebind { 269 using other = scalable_allocator<U>; 270 }; 271 }; 272 #endif 273 274 template<typename T, typename U> 275 inline bool operator==(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return true; } 276 277 #if !__TBB_CPP20_COMPARISONS_PRESENT 278 template<typename T, typename U> 279 inline bool operator!=(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return false; } 280 #endif 281 282 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT 283 284 //! C++17 memory resource implementation for scalable allocator 285 //! ISO C++ Section 23.12.2 286 class scalable_resource_impl : public std::pmr::memory_resource { 287 private: 288 void* do_allocate(std::size_t bytes, std::size_t alignment) override { 289 void* p = scalable_aligned_malloc(bytes, alignment); 290 if (!p) { 291 throw_exception(std::bad_alloc()); 292 } 293 return p; 294 } 295 296 void do_deallocate(void* ptr, std::size_t /*bytes*/, std::size_t /*alignment*/) override { 297 scalable_free(ptr); 298 } 299 300 //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any 301 //! other instance of this class 302 bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { 303 return this == &other || 304 #if __TBB_USE_OPTIONAL_RTTI 305 dynamic_cast<const scalable_resource_impl*>(&other) != nullptr; 306 #else 307 false; 308 #endif 309 } 310 }; 311 312 //! Global scalable allocator memory resource provider 313 inline std::pmr::memory_resource* scalable_memory_resource() noexcept { 314 static tbb::detail::d1::scalable_resource_impl scalable_res; 315 return &scalable_res; 316 } 317 318 #endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT 319 320 } // namespace d1 321 } // namespace detail 322 323 inline namespace v1 { 324 using detail::d1::scalable_allocator; 325 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT 326 using detail::d1::scalable_memory_resource; 327 #endif 328 } // namespace v1 329 330 } // namespace tbb 331 332 #endif /* __cplusplus */ 333 334 #endif /* __TBB_scalable_allocator_H */ 335