1 /* 2 Copyright (c) 2005-2021 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #include "oneapi/tbb/version.h" 18 19 #include "oneapi/tbb/detail/_exception.h" 20 #include "oneapi/tbb/detail/_assert.h" 21 #include "oneapi/tbb/detail/_utils.h" 22 #include "oneapi/tbb/tbb_allocator.h" // Is this OK? 23 #include "oneapi/tbb/cache_aligned_allocator.h" 24 25 #include "dynamic_link.h" 26 #include "misc.h" 27 28 #include <cstdlib> 29 30 #ifdef _WIN32 31 #include <windows.h> 32 #else 33 #include <dlfcn.h> 34 #endif 35 36 #if __TBB_WEAK_SYMBOLS_PRESENT 37 38 #pragma weak scalable_malloc 39 #pragma weak scalable_free 40 #pragma weak scalable_aligned_malloc 41 #pragma weak scalable_aligned_free 42 43 extern "C" { 44 void* scalable_malloc(std::size_t); 45 void scalable_free(void*); 46 void* scalable_aligned_malloc(std::size_t, std::size_t); 47 void scalable_aligned_free(void*); 48 } 49 50 #endif /* __TBB_WEAK_SYMBOLS_PRESENT */ 51 52 namespace tbb { 53 namespace detail { 54 namespace r1 { 55 56 //! Initialization routine used for first indirect call via allocate_handler. 57 static void* initialize_allocate_handler(std::size_t size); 58 59 //! Handler for memory allocation 60 using allocate_handler_type = void* (*)(std::size_t size); 61 static std::atomic<allocate_handler_type> allocate_handler{ &initialize_allocate_handler }; 62 allocate_handler_type allocate_handler_unsafe = nullptr; 63 64 //! Handler for memory deallocation 65 static void (*deallocate_handler)(void* pointer) = nullptr; 66 67 //! Initialization routine used for first indirect call via cache_aligned_allocate_handler. 68 static void* initialize_cache_aligned_allocate_handler(std::size_t n, std::size_t alignment); 69 70 //! Allocates memory using standard malloc. It is used when scalable_allocator is not available 71 static void* std_cache_aligned_allocate(std::size_t n, std::size_t alignment); 72 73 //! Allocates memory using standard free. It is used when scalable_allocator is not available 74 static void std_cache_aligned_deallocate(void* p); 75 76 //! Handler for padded memory allocation 77 using cache_aligned_allocate_handler_type = void* (*)(std::size_t n, std::size_t alignment); 78 static std::atomic<cache_aligned_allocate_handler_type> cache_aligned_allocate_handler{ &initialize_cache_aligned_allocate_handler }; 79 cache_aligned_allocate_handler_type cache_aligned_allocate_handler_unsafe = nullptr; 80 81 //! Handler for padded memory deallocation 82 static void (*cache_aligned_deallocate_handler)(void* p) = nullptr; 83 84 //! Table describing how to link the handlers. 85 static const dynamic_link_descriptor MallocLinkTable[] = { 86 DLD(scalable_malloc, allocate_handler_unsafe), 87 DLD(scalable_free, deallocate_handler), 88 DLD(scalable_aligned_malloc, cache_aligned_allocate_handler_unsafe), 89 DLD(scalable_aligned_free, cache_aligned_deallocate_handler), 90 }; 91 92 93 #if TBB_USE_DEBUG 94 #define DEBUG_SUFFIX "_debug" 95 #else 96 #define DEBUG_SUFFIX 97 #endif /* TBB_USE_DEBUG */ 98 99 // MALLOCLIB_NAME is the name of the oneTBB memory allocator library. 100 #if _WIN32||_WIN64 101 #define MALLOCLIB_NAME "tbbmalloc" DEBUG_SUFFIX ".dll" 102 #elif __APPLE__ 103 #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".dylib" 104 #elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__ 105 #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so" 106 #elif __unix__ // Note that order of these #elif's is important! 107 #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so.2" 108 #else 109 #error Unknown OS 110 #endif 111 112 //! Initialize the allocation/free handler pointers. 113 /** Caller is responsible for ensuring this routine is called exactly once. 114 The routine attempts to dynamically link with the TBB memory allocator. 115 If that allocator is not found, it links to malloc and free. */ 116 void initialize_handler_pointers() { 117 __TBB_ASSERT(allocate_handler == &initialize_allocate_handler, NULL); 118 bool success = dynamic_link(MALLOCLIB_NAME, MallocLinkTable, 4); 119 if(!success) { 120 // If unsuccessful, set the handlers to the default routines. 121 // This must be done now, and not before FillDynamicLinks runs, because if other 122 // threads call the handlers, we want them to go through the DoOneTimeInitializations logic, 123 // which forces them to wait. 124 allocate_handler_unsafe = &std::malloc; 125 deallocate_handler = &std::free; 126 cache_aligned_allocate_handler_unsafe = &std_cache_aligned_allocate; 127 cache_aligned_deallocate_handler = &std_cache_aligned_deallocate; 128 } 129 130 allocate_handler.store(allocate_handler_unsafe, std::memory_order_release); 131 cache_aligned_allocate_handler.store(cache_aligned_allocate_handler_unsafe, std::memory_order_release); 132 133 PrintExtraVersionInfo( "ALLOCATOR", success?"scalable_malloc":"malloc" ); 134 } 135 136 static std::once_flag initialization_state; 137 void initialize_cache_aligned_allocator() { 138 std::call_once(initialization_state, &initialize_handler_pointers); 139 } 140 141 //! Executed on very first call through allocate_handler 142 static void* initialize_allocate_handler(std::size_t size) { 143 initialize_cache_aligned_allocator(); 144 __TBB_ASSERT(allocate_handler != &initialize_allocate_handler, NULL); 145 return (*allocate_handler)(size); 146 } 147 148 //! Executed on very first call through cache_aligned_allocate_handler 149 static void* initialize_cache_aligned_allocate_handler(std::size_t bytes, std::size_t alignment) { 150 initialize_cache_aligned_allocator(); 151 __TBB_ASSERT(cache_aligned_allocate_handler != &initialize_cache_aligned_allocate_handler, NULL); 152 return (*cache_aligned_allocate_handler)(bytes, alignment); 153 } 154 155 // TODO: use CPUID to find actual line size, though consider backward compatibility 156 // nfs - no false sharing 157 static constexpr std::size_t nfs_size = 128; 158 159 std::size_t __TBB_EXPORTED_FUNC cache_line_size() { 160 return nfs_size; 161 } 162 163 void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size) { 164 const std::size_t cache_line_size = nfs_size; 165 __TBB_ASSERT(is_power_of_two(cache_line_size), "must be power of two"); 166 167 // Check for overflow 168 if (size + cache_line_size < size) { 169 throw_exception(exception_id::bad_alloc); 170 } 171 // scalable_aligned_malloc considers zero size request an error, and returns NULL 172 if (size == 0) size = 1; 173 174 void* result = cache_aligned_allocate_handler.load(std::memory_order_acquire)(size, cache_line_size); 175 if (!result) { 176 throw_exception(exception_id::bad_alloc); 177 } 178 __TBB_ASSERT(is_aligned(result, cache_line_size), "The returned address isn't aligned"); 179 return result; 180 } 181 182 void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p) { 183 __TBB_ASSERT(cache_aligned_deallocate_handler, "Initialization has not been yet."); 184 (*cache_aligned_deallocate_handler)(p); 185 } 186 187 static void* std_cache_aligned_allocate(std::size_t bytes, std::size_t alignment) { 188 // TODO: make it common with cache_aligned_resource 189 std::size_t space = alignment + bytes; 190 std::uintptr_t base = reinterpret_cast<std::uintptr_t>(std::malloc(space)); 191 if (!base) { 192 return nullptr; 193 } 194 std::uintptr_t result = (base + nfs_size) & ~(nfs_size - 1); 195 // Round up to the next cache line (align the base address) 196 __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), "Cannot store a base pointer to the header"); 197 __TBB_ASSERT(space - (result - base) >= bytes, "Not enough space for the storage"); 198 199 // Record where block actually starts. 200 (reinterpret_cast<std::uintptr_t*>(result))[-1] = base; 201 return reinterpret_cast<void*>(result); 202 } 203 204 static void std_cache_aligned_deallocate(void* p) { 205 if (p) { 206 __TBB_ASSERT(reinterpret_cast<std::uintptr_t>(p) >= 0x4096, "attempt to free block not obtained from cache_aligned_allocator"); 207 // Recover where block actually starts 208 std::uintptr_t base = (reinterpret_cast<std::uintptr_t*>(p))[-1]; 209 __TBB_ASSERT(((base + nfs_size) & ~(nfs_size - 1)) == reinterpret_cast<std::uintptr_t>(p), "Incorrect alignment or not allocated by std_cache_aligned_deallocate?"); 210 std::free(reinterpret_cast<void*>(base)); 211 } 212 } 213 214 void* __TBB_EXPORTED_FUNC allocate_memory(std::size_t size) { 215 void* result = allocate_handler.load(std::memory_order_acquire)(size); 216 if (!result) { 217 throw_exception(exception_id::bad_alloc); 218 } 219 return result; 220 } 221 222 void __TBB_EXPORTED_FUNC deallocate_memory(void* p) { 223 if (p) { 224 __TBB_ASSERT(deallocate_handler, "Initialization has not been yet."); 225 (*deallocate_handler)(p); 226 } 227 } 228 229 bool __TBB_EXPORTED_FUNC is_tbbmalloc_used() { 230 auto handler_snapshot = allocate_handler.load(std::memory_order_acquire); 231 if (handler_snapshot == &initialize_allocate_handler) { 232 initialize_cache_aligned_allocator(); 233 } 234 handler_snapshot = allocate_handler.load(std::memory_order_relaxed); 235 __TBB_ASSERT(handler_snapshot != &initialize_allocate_handler && deallocate_handler != nullptr, NULL); 236 // Cast to void avoids type mismatch errors on some compilers (e.g. __IBMCPP__) 237 __TBB_ASSERT((reinterpret_cast<void*>(handler_snapshot) == reinterpret_cast<void*>(&std::malloc)) == (reinterpret_cast<void*>(deallocate_handler) == reinterpret_cast<void*>(&std::free)), 238 "Both shim pointers must refer to routines from the same package (either TBB or CRT)"); 239 return reinterpret_cast<void*>(handler_snapshot) == reinterpret_cast<void*>(&std::malloc); 240 } 241 242 } // namespace r1 243 } // namespace detail 244 } // namespace tbb 245