1 /* 2 Copyright (c) 2005-2021 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #include "oneapi/tbb/version.h" 18 19 #include "oneapi/tbb/detail/_exception.h" 20 #include "oneapi/tbb/detail/_assert.h" 21 #include "oneapi/tbb/detail/_utils.h" 22 23 #include "dynamic_link.h" 24 #include "misc.h" 25 26 #include <cstdlib> 27 28 #if _WIN32 || _WIN64 29 #include <Windows.h> 30 #else 31 #include <dlfcn.h> 32 #endif /* _WIN32||_WIN64 */ 33 34 #if __TBB_WEAK_SYMBOLS_PRESENT 35 36 #pragma weak scalable_malloc 37 #pragma weak scalable_free 38 #pragma weak scalable_aligned_malloc 39 #pragma weak scalable_aligned_free 40 41 extern "C" { 42 void* scalable_malloc(std::size_t); 43 void scalable_free(void*); 44 void* scalable_aligned_malloc(std::size_t, std::size_t); 45 void scalable_aligned_free(void*); 46 } 47 48 #endif /* __TBB_WEAK_SYMBOLS_PRESENT */ 49 50 namespace tbb { 51 namespace detail { 52 namespace r1 { 53 54 //! Initialization routine used for first indirect call via allocate_handler. 55 static void* initialize_allocate_handler(std::size_t size); 56 57 //! Handler for memory allocation 58 using allocate_handler_type = void* (*)(std::size_t size); 59 static std::atomic<allocate_handler_type> allocate_handler{ &initialize_allocate_handler }; 60 allocate_handler_type allocate_handler_unsafe = nullptr; 61 62 //! Handler for memory deallocation 63 static void (*deallocate_handler)(void* pointer) = nullptr; 64 65 //! Initialization routine used for first indirect call via cache_aligned_allocate_handler. 66 static void* initialize_cache_aligned_allocate_handler(std::size_t n, std::size_t alignment); 67 68 //! Allocates memory using standard malloc. It is used when scalable_allocator is not available 69 static void* std_cache_aligned_allocate(std::size_t n, std::size_t alignment); 70 71 //! Allocates memory using standard free. It is used when scalable_allocator is not available 72 static void std_cache_aligned_deallocate(void* p); 73 74 //! Handler for padded memory allocation 75 using cache_aligned_allocate_handler_type = void* (*)(std::size_t n, std::size_t alignment); 76 static std::atomic<cache_aligned_allocate_handler_type> cache_aligned_allocate_handler{ &initialize_cache_aligned_allocate_handler }; 77 cache_aligned_allocate_handler_type cache_aligned_allocate_handler_unsafe = nullptr; 78 79 //! Handler for padded memory deallocation 80 static void (*cache_aligned_deallocate_handler)(void* p) = nullptr; 81 82 //! Table describing how to link the handlers. 83 static const dynamic_link_descriptor MallocLinkTable[] = { 84 DLD(scalable_malloc, allocate_handler_unsafe), 85 DLD(scalable_free, deallocate_handler), 86 DLD(scalable_aligned_malloc, cache_aligned_allocate_handler_unsafe), 87 DLD(scalable_aligned_free, cache_aligned_deallocate_handler), 88 }; 89 90 91 #if TBB_USE_DEBUG 92 #define DEBUG_SUFFIX "_debug" 93 #else 94 #define DEBUG_SUFFIX 95 #endif /* TBB_USE_DEBUG */ 96 97 // MALLOCLIB_NAME is the name of the oneTBB memory allocator library. 98 #if _WIN32||_WIN64 99 #define MALLOCLIB_NAME "tbbmalloc" DEBUG_SUFFIX ".dll" 100 #elif __APPLE__ 101 #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".dylib" 102 #elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__ 103 #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so" 104 #elif __unix__ // Note that order of these #elif's is important! 105 #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so.2" 106 #else 107 #error Unknown OS 108 #endif 109 110 //! Initialize the allocation/free handler pointers. 111 /** Caller is responsible for ensuring this routine is called exactly once. 112 The routine attempts to dynamically link with the TBB memory allocator. 113 If that allocator is not found, it links to malloc and free. */ 114 void initialize_handler_pointers() { 115 __TBB_ASSERT(allocate_handler == &initialize_allocate_handler, NULL); 116 bool success = dynamic_link(MALLOCLIB_NAME, MallocLinkTable, 4); 117 if(!success) { 118 // If unsuccessful, set the handlers to the default routines. 119 // This must be done now, and not before FillDynamicLinks runs, because if other 120 // threads call the handlers, we want them to go through the DoOneTimeInitializations logic, 121 // which forces them to wait. 122 allocate_handler_unsafe = &std::malloc; 123 deallocate_handler = &std::free; 124 cache_aligned_allocate_handler_unsafe = &std_cache_aligned_allocate; 125 cache_aligned_deallocate_handler = &std_cache_aligned_deallocate; 126 } 127 128 allocate_handler.store(allocate_handler_unsafe, std::memory_order_release); 129 cache_aligned_allocate_handler.store(cache_aligned_allocate_handler_unsafe, std::memory_order_release); 130 131 PrintExtraVersionInfo( "ALLOCATOR", success?"scalable_malloc":"malloc" ); 132 } 133 134 static std::once_flag initialization_state; 135 void initialize_cache_aligned_allocator() { 136 std::call_once(initialization_state, &initialize_handler_pointers); 137 } 138 139 //! Executed on very first call through allocate_handler 140 static void* initialize_allocate_handler(std::size_t size) { 141 initialize_cache_aligned_allocator(); 142 __TBB_ASSERT(allocate_handler != &initialize_allocate_handler, NULL); 143 return (*allocate_handler)(size); 144 } 145 146 //! Executed on very first call through cache_aligned_allocate_handler 147 static void* initialize_cache_aligned_allocate_handler(std::size_t bytes, std::size_t alignment) { 148 initialize_cache_aligned_allocator(); 149 __TBB_ASSERT(cache_aligned_allocate_handler != &initialize_cache_aligned_allocate_handler, NULL); 150 return (*cache_aligned_allocate_handler)(bytes, alignment); 151 } 152 153 // TODO: use CPUID to find actual line size, though consider backward compatibility 154 // nfs - no false sharing 155 static constexpr std::size_t nfs_size = 128; 156 157 std::size_t __TBB_EXPORTED_FUNC cache_line_size() { 158 return nfs_size; 159 } 160 161 void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size) { 162 const std::size_t cache_line_size = nfs_size; 163 __TBB_ASSERT(is_power_of_two(cache_line_size), "must be power of two"); 164 165 // Check for overflow 166 if (size + cache_line_size < size) { 167 throw_exception(exception_id::bad_alloc); 168 } 169 // scalable_aligned_malloc considers zero size request an error, and returns NULL 170 if (size == 0) size = 1; 171 172 void* result = cache_aligned_allocate_handler.load(std::memory_order_acquire)(size, cache_line_size); 173 if (!result) { 174 throw_exception(exception_id::bad_alloc); 175 } 176 __TBB_ASSERT(is_aligned(result, cache_line_size), "The returned address isn't aligned"); 177 return result; 178 } 179 180 void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p) { 181 __TBB_ASSERT(cache_aligned_deallocate_handler, "Initialization has not been yet."); 182 (*cache_aligned_deallocate_handler)(p); 183 } 184 185 static void* std_cache_aligned_allocate(std::size_t bytes, std::size_t alignment) { 186 // TODO: make it common with cache_aligned_resource 187 std::size_t space = alignment + bytes; 188 std::uintptr_t base = reinterpret_cast<std::uintptr_t>(std::malloc(space)); 189 if (!base) { 190 return nullptr; 191 } 192 std::uintptr_t result = (base + nfs_size) & ~(nfs_size - 1); 193 // Round up to the next cache line (align the base address) 194 __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), "Cannot store a base pointer to the header"); 195 __TBB_ASSERT(space - (result - base) >= bytes, "Not enough space for the storage"); 196 197 // Record where block actually starts. 198 (reinterpret_cast<std::uintptr_t*>(result))[-1] = base; 199 return reinterpret_cast<void*>(result); 200 } 201 202 static void std_cache_aligned_deallocate(void* p) { 203 if (p) { 204 __TBB_ASSERT(reinterpret_cast<std::uintptr_t>(p) >= 0x4096, "attempt to free block not obtained from cache_aligned_allocator"); 205 // Recover where block actually starts 206 std::uintptr_t base = (reinterpret_cast<std::uintptr_t*>(p))[-1]; 207 __TBB_ASSERT(((base + nfs_size) & ~(nfs_size - 1)) == reinterpret_cast<std::uintptr_t>(p), "Incorrect alignment or not allocated by std_cache_aligned_deallocate?"); 208 std::free(reinterpret_cast<void*>(base)); 209 } 210 } 211 212 void* __TBB_EXPORTED_FUNC allocate_memory(std::size_t size) { 213 void* result = allocate_handler.load(std::memory_order_acquire)(size); 214 if (!result) { 215 throw_exception(exception_id::bad_alloc); 216 } 217 return result; 218 } 219 220 void __TBB_EXPORTED_FUNC deallocate_memory(void* p) { 221 if (p) { 222 __TBB_ASSERT(deallocate_handler, "Initialization has not been yet."); 223 (*deallocate_handler)(p); 224 } 225 } 226 227 bool __TBB_EXPORTED_FUNC is_tbbmalloc_used() { 228 auto handler_snapshot = allocate_handler.load(std::memory_order_acquire); 229 if (handler_snapshot == &initialize_allocate_handler) { 230 initialize_cache_aligned_allocator(); 231 } 232 handler_snapshot = allocate_handler.load(std::memory_order_relaxed); 233 __TBB_ASSERT(handler_snapshot != &initialize_allocate_handler && deallocate_handler != nullptr, NULL); 234 // Cast to void avoids type mismatch errors on some compilers (e.g. __IBMCPP__) 235 __TBB_ASSERT((reinterpret_cast<void*>(handler_snapshot) == reinterpret_cast<void*>(&std::malloc)) == (reinterpret_cast<void*>(deallocate_handler) == reinterpret_cast<void*>(&std::free)), 236 "Both shim pointers must refer to routines from the same package (either TBB or CRT)"); 237 return reinterpret_cast<void*>(handler_snapshot) == reinterpret_cast<void*>(&std::malloc); 238 } 239 240 } // namespace r1 241 } // namespace detail 242 } // namespace tbb 243