1 /* 2 Copyright (c) 2005-2020 Intel Corporation 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 #include "oneapi/tbb/version.h" 18 19 #include "oneapi/tbb/detail/_exception.h" 20 #include "oneapi/tbb/detail/_assert.h" 21 #include "oneapi/tbb/detail/_utils.h" 22 23 #include "dynamic_link.h" 24 #include "misc.h" 25 26 #include <cstdlib> 27 28 #if _WIN32 || _WIN64 29 #include <Windows.h> 30 #else 31 #include <dlfcn.h> 32 #endif /* _WIN32||_WIN64 */ 33 34 #if __TBB_WEAK_SYMBOLS_PRESENT 35 36 #pragma weak scalable_malloc 37 #pragma weak scalable_free 38 #pragma weak scalable_aligned_malloc 39 #pragma weak scalable_aligned_free 40 41 extern "C" { 42 void* scalable_malloc(std::size_t); 43 void scalable_free(void*); 44 void* scalable_aligned_malloc(std::size_t, std::size_t); 45 void scalable_aligned_free(void*); 46 } 47 48 #endif /* __TBB_WEAK_SYMBOLS_PRESENT */ 49 50 namespace tbb { 51 namespace detail { 52 namespace r1 { 53 54 //! Initialization routine used for first indirect call via allocate_handler. 55 static void* initialize_allocate_handler(std::size_t size); 56 57 //! Handler for memory allocation 58 static void* (*allocate_handler)(std::size_t size) = &initialize_allocate_handler; 59 60 //! Handler for memory deallocation 61 static void (*deallocate_handler)(void* pointer) = nullptr; 62 63 //! Initialization routine used for first indirect call via cache_aligned_allocate_handler. 64 static void* initialize_cache_aligned_allocate_handler(std::size_t n, std::size_t alignment); 65 66 //! Allocates memory using standard malloc. It is used when scalable_allocator is not available 67 static void* std_cache_aligned_allocate(std::size_t n, std::size_t alignment); 68 69 //! Allocates memory using standard free. It is used when scalable_allocator is not available 70 static void std_cache_aligned_deallocate(void* p); 71 72 //! Handler for padded memory allocation 73 static void* (*cache_aligned_allocate_handler)(std::size_t n, std::size_t alignment) = &initialize_cache_aligned_allocate_handler; 74 75 //! Handler for padded memory deallocation 76 static void (*cache_aligned_deallocate_handler)(void* p) = nullptr; 77 78 //! Table describing how to link the handlers. 79 static const dynamic_link_descriptor MallocLinkTable[] = { 80 DLD(scalable_malloc, allocate_handler), 81 DLD(scalable_free, deallocate_handler), 82 DLD(scalable_aligned_malloc, cache_aligned_allocate_handler), 83 DLD(scalable_aligned_free, cache_aligned_deallocate_handler), 84 }; 85 86 87 #if TBB_USE_DEBUG 88 #define DEBUG_SUFFIX "_debug" 89 #else 90 #define DEBUG_SUFFIX 91 #endif /* TBB_USE_DEBUG */ 92 93 // MALLOCLIB_NAME is the name of the oneTBB memory allocator library. 94 #if _WIN32||_WIN64 95 #define MALLOCLIB_NAME "tbbmalloc" DEBUG_SUFFIX ".dll" 96 #elif __APPLE__ 97 #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".dylib" 98 #elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__ 99 #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so" 100 #elif __linux__ // Note that order of these #elif's is important! 101 #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so.2" 102 #else 103 #error Unknown OS 104 #endif 105 106 //! Initialize the allocation/free handler pointers. 107 /** Caller is responsible for ensuring this routine is called exactly once. 108 The routine attempts to dynamically link with the TBB memory allocator. 109 If that allocator is not found, it links to malloc and free. */ 110 void initialize_handler_pointers() { 111 __TBB_ASSERT(allocate_handler == &initialize_allocate_handler, NULL); 112 bool success = dynamic_link(MALLOCLIB_NAME, MallocLinkTable, 4); 113 if(!success) { 114 // If unsuccessful, set the handlers to the default routines. 115 // This must be done now, and not before FillDynamicLinks runs, because if other 116 // threads call the handlers, we want them to go through the DoOneTimeInitializations logic, 117 // which forces them to wait. 118 allocate_handler = &std::malloc; 119 deallocate_handler = &std::free; 120 cache_aligned_allocate_handler = &std_cache_aligned_allocate; 121 cache_aligned_deallocate_handler = &std_cache_aligned_deallocate; 122 } 123 124 PrintExtraVersionInfo( "ALLOCATOR", success?"scalable_malloc":"malloc" ); 125 } 126 127 static std::once_flag initialization_state; 128 void initialize_cache_aligned_allocator() { 129 std::call_once(initialization_state, &initialize_handler_pointers); 130 } 131 132 //! Executed on very first call through allocate_handler 133 static void* initialize_allocate_handler(std::size_t size) { 134 initialize_cache_aligned_allocator(); 135 __TBB_ASSERT(allocate_handler != &initialize_allocate_handler, NULL); 136 return (*allocate_handler)(size); 137 } 138 139 //! Executed on very first call through cache_aligned_allocate_handler 140 static void* initialize_cache_aligned_allocate_handler(std::size_t bytes, std::size_t alignment) { 141 initialize_cache_aligned_allocator(); 142 __TBB_ASSERT(cache_aligned_allocate_handler != &initialize_cache_aligned_allocate_handler, NULL); 143 return (*cache_aligned_allocate_handler)(bytes, alignment); 144 } 145 146 // TODO: use CPUID to find actual line size, though consider backward compatibility 147 // nfs - no false sharing 148 static constexpr std::size_t nfs_size = 128; 149 150 std::size_t __TBB_EXPORTED_FUNC cache_line_size() { 151 return nfs_size; 152 } 153 154 void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size) { 155 const std::size_t cache_line_size = nfs_size; 156 __TBB_ASSERT(is_power_of_two(cache_line_size), "must be power of two"); 157 158 // Check for overflow 159 if (size + cache_line_size < size) { 160 throw_exception(exception_id::bad_alloc); 161 } 162 // scalable_aligned_malloc considers zero size request an error, and returns NULL 163 if (size == 0) size = 1; 164 165 void* result = cache_aligned_allocate_handler(size, cache_line_size); 166 if (!result) { 167 throw_exception(exception_id::bad_alloc); 168 } 169 __TBB_ASSERT(is_aligned(result, cache_line_size), "The returned address isn't aligned"); 170 return result; 171 } 172 173 void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p) { 174 __TBB_ASSERT(cache_aligned_deallocate_handler, "Initialization has not been yet."); 175 (*cache_aligned_deallocate_handler)(p); 176 } 177 178 static void* std_cache_aligned_allocate(std::size_t bytes, std::size_t alignment) { 179 // TODO: make it common with cache_aligned_resource 180 std::size_t space = alignment + bytes; 181 std::uintptr_t base = reinterpret_cast<std::uintptr_t>(std::malloc(space)); 182 if (!base) { 183 return nullptr; 184 } 185 std::uintptr_t result = (base + nfs_size) & ~(nfs_size - 1); 186 // Round up to the next cache line (align the base address) 187 __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), "Cannot store a base pointer to the header"); 188 __TBB_ASSERT(space - (result - base) >= bytes, "Not enough space for the storage"); 189 190 // Record where block actually starts. 191 (reinterpret_cast<std::uintptr_t*>(result))[-1] = base; 192 return reinterpret_cast<void*>(result); 193 } 194 195 static void std_cache_aligned_deallocate(void* p) { 196 if (p) { 197 __TBB_ASSERT(reinterpret_cast<std::uintptr_t>(p) >= 0x4096, "attempt to free block not obtained from cache_aligned_allocator"); 198 // Recover where block actually starts 199 std::uintptr_t base = (reinterpret_cast<std::uintptr_t*>(p))[-1]; 200 __TBB_ASSERT(((base + nfs_size) & ~(nfs_size - 1)) == reinterpret_cast<std::uintptr_t>(p), "Incorrect alignment or not allocated by std_cache_aligned_deallocate?"); 201 std::free(reinterpret_cast<void*>(base)); 202 } 203 } 204 205 void* __TBB_EXPORTED_FUNC allocate_memory(std::size_t size) { 206 void* result = (*allocate_handler)(size); 207 if (!result) { 208 throw_exception(exception_id::bad_alloc); 209 } 210 return result; 211 } 212 213 void __TBB_EXPORTED_FUNC deallocate_memory(void* p) { 214 if (p) { 215 __TBB_ASSERT(deallocate_handler, "Initialization has not been yet."); 216 (*deallocate_handler)(p); 217 } 218 } 219 220 bool __TBB_EXPORTED_FUNC is_tbbmalloc_used() { 221 if (allocate_handler == &initialize_allocate_handler) { 222 void* void_ptr = allocate_handler(1); 223 deallocate_handler(void_ptr); 224 } 225 __TBB_ASSERT(allocate_handler != &initialize_allocate_handler && deallocate_handler != nullptr, NULL); 226 // Cast to void avoids type mismatch errors on some compilers (e.g. __IBMCPP__) 227 __TBB_ASSERT((reinterpret_cast<void*>(allocate_handler) == reinterpret_cast<void*>(&std::malloc)) == (reinterpret_cast<void*>(deallocate_handler) == reinterpret_cast<void*>(&std::free)), 228 "Both shim pointers must refer to routines from the same package (either TBB or CRT)"); 229 return reinterpret_cast<void*>(allocate_handler) == reinterpret_cast<void*>(&std::malloc); 230 } 231 232 } // namespace r1 233 } // namespace detail 234 } // namespace tbb 235