1 /*
2 Copyright (c) 2005-2022 Intel Corporation
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 #if __unix__ && !__ANDROID__
18 // include <bits/c++config.h> indirectly so that <cstdlib> is not included
19 #include <cstddef>
20 // include <features.h> indirectly so that <stdlib.h> is not included
21 #include <unistd.h>
22 // Working around compiler issue with Anaconda's gcc 7.3 compiler package.
23 // New gcc ported for old libc may provide their inline implementation
24 // of aligned_alloc as required by new C++ standard, this makes it hard to
25 // redefine aligned_alloc here. However, running on systems with new libc
26 // version, it still needs it to be redefined, thus tricking system headers
27 #if defined(__GLIBC_PREREQ)
28 #if !__GLIBC_PREREQ(2, 16) && _GLIBCXX_HAVE_ALIGNED_ALLOC
29 // tell <cstdlib> that there is no aligned_alloc
30 #undef _GLIBCXX_HAVE_ALIGNED_ALLOC
31 // trick <stdlib.h> to define another symbol instead
32 #define aligned_alloc __hidden_redefined_aligned_alloc
33 // Fix the state and undefine the trick
34 #include <cstdlib>
35 #undef aligned_alloc
36 #endif // !__GLIBC_PREREQ(2, 16) && _GLIBCXX_HAVE_ALIGNED_ALLOC
37 #endif // defined(__GLIBC_PREREQ)
38 #include <cstdlib>
39 #endif // __unix__ && !__ANDROID__
40
41 #include "proxy.h"
42
43 #include "oneapi/tbb/detail/_config.h"
44 #include "oneapi/tbb/scalable_allocator.h"
45 #include "../tbb/environment.h"
46
47 #if !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) && !defined(__SUNPRO_CC)
48 #if TBB_USE_EXCEPTIONS
49 #error Compilation settings do not support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0.
50 #elif !defined(TBB_USE_EXCEPTIONS)
51 #define TBB_USE_EXCEPTIONS 0
52 #endif
53 #elif !defined(TBB_USE_EXCEPTIONS)
54 #define TBB_USE_EXCEPTIONS 1
55 #endif
56
57 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT
58 /*** internal global operator new implementation (Linux, Windows) ***/
59 #include <new>
60
61 // Synchronization primitives to protect original library pointers and new_handler
62 #include "../tbbmalloc/Synchronize.h"
63 // Use MallocMutex implementation
64 typedef MallocMutex ProxyMutex;
65
66 // Adds aliasing and copy attributes to function if available
67 #if defined(__has_attribute)
68 #if __has_attribute(__copy__)
69 #define __TBB_ALIAS_ATTR_COPY(name) __attribute__((alias (#name), __copy__(name)))
70 #endif
71 #endif
72
73 #ifndef __TBB_ALIAS_ATTR_COPY
74 #define __TBB_ALIAS_ATTR_COPY(name) __attribute__((alias (#name)))
75 #endif
76
77 // In case there is no std::get_new_handler function
78 // which provides synchronized access to std::new_handler
79 #if !__TBB_CPP11_GET_NEW_HANDLER_PRESENT
80 static ProxyMutex new_lock;
81 #endif
82
InternalOperatorNew(size_t sz)83 static inline void* InternalOperatorNew(size_t sz) {
84 void* res = scalable_malloc(sz);
85 #if TBB_USE_EXCEPTIONS
86 while (!res) {
87 std::new_handler handler;
88 #if __TBB_CPP11_GET_NEW_HANDLER_PRESENT
89 handler = std::get_new_handler();
90 #else
91 {
92 ProxyMutex::scoped_lock lock(new_lock);
93 handler = std::set_new_handler(0);
94 std::set_new_handler(handler);
95 }
96 #endif
97 if (handler) {
98 (*handler)();
99 } else {
100 throw std::bad_alloc();
101 }
102 res = scalable_malloc(sz);
103 }
104 #endif /* TBB_USE_EXCEPTIONS */
105 return res;
106 }
107 /*** end of internal global operator new implementation ***/
108 #endif // MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT
109
110 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED
111
112 #ifndef __THROW
113 #define __THROW
114 #endif
115
116 /*** service functions and variables ***/
117 #include <string.h> // for memset
118 #include <unistd.h> // for sysconf
119
120 static long memoryPageSize;
121
initPageSize()122 static inline void initPageSize()
123 {
124 memoryPageSize = sysconf(_SC_PAGESIZE);
125 }
126
127 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED
128 #include <dlfcn.h>
129 #include <malloc.h> // mallinfo
130
131 /* __TBB_malloc_proxy used as a weak symbol by libtbbmalloc for:
132 1) detection that the proxy library is loaded
133 2) check that dlsym("malloc") found something different from our replacement malloc
134 */
135
136 extern "C" void *__TBB_malloc_proxy(size_t) __TBB_ALIAS_ATTR_COPY(malloc);
137
138 static void *orig_msize;
139
140 #elif MALLOC_ZONE_OVERLOAD_ENABLED
141
142 #include "proxy_overload_osx.h"
143
144 #endif // MALLOC_ZONE_OVERLOAD_ENABLED
145
146 // Original (i.e., replaced) functions,
147 // they are never changed for MALLOC_ZONE_OVERLOAD_ENABLED.
148 static void *orig_free,
149 *orig_realloc;
150
151 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED
152 #define ZONE_ARG
153 #define PREFIX(name) name
154
155 static void *orig_libc_free,
156 *orig_libc_realloc;
157
158 // We already tried to find ptr to original functions.
159 static std::atomic<bool> origFuncSearched{false};
160
InitOrigPointers()161 inline void InitOrigPointers()
162 {
163 // race is OK here, as different threads found same functions
164 if (!origFuncSearched.load(std::memory_order_acquire)) {
165 orig_free = dlsym(RTLD_NEXT, "free");
166 orig_realloc = dlsym(RTLD_NEXT, "realloc");
167 orig_msize = dlsym(RTLD_NEXT, "malloc_usable_size");
168 orig_libc_free = dlsym(RTLD_NEXT, "__libc_free");
169 orig_libc_realloc = dlsym(RTLD_NEXT, "__libc_realloc");
170
171 origFuncSearched.store(true, std::memory_order_release);
172 }
173 }
174
175 /*** replacements for malloc and the family ***/
176 extern "C" {
177 #elif MALLOC_ZONE_OVERLOAD_ENABLED
178
179 // each impl_* function has such 1st argument, it's unused
180 #define ZONE_ARG struct _malloc_zone_t *,
181 #define PREFIX(name) impl_##name
182 // not interested in original functions for zone overload
183 inline void InitOrigPointers() {}
184
185 #endif // MALLOC_UNIXLIKE_OVERLOAD_ENABLED and MALLOC_ZONE_OVERLOAD_ENABLED
186
PREFIX(malloc)187 void *PREFIX(malloc)(ZONE_ARG size_t size) __THROW
188 {
189 return scalable_malloc(size);
190 }
191
PREFIX(calloc)192 void *PREFIX(calloc)(ZONE_ARG size_t num, size_t size) __THROW
193 {
194 return scalable_calloc(num, size);
195 }
196
PREFIX(free)197 void PREFIX(free)(ZONE_ARG void *object) __THROW
198 {
199 InitOrigPointers();
200 __TBB_malloc_safer_free(object, (void (*)(void*))orig_free);
201 }
202
PREFIX(realloc)203 void *PREFIX(realloc)(ZONE_ARG void* ptr, size_t sz) __THROW
204 {
205 InitOrigPointers();
206 return __TBB_malloc_safer_realloc(ptr, sz, orig_realloc);
207 }
208
209 /* The older *NIX interface for aligned allocations;
210 it's formally substituted by posix_memalign and deprecated,
211 so we do not expect it to cause cyclic dependency with C RTL. */
PREFIX(memalign)212 void *PREFIX(memalign)(ZONE_ARG size_t alignment, size_t size) __THROW
213 {
214 return scalable_aligned_malloc(size, alignment);
215 }
216
217 /* valloc allocates memory aligned on a page boundary */
PREFIX(valloc)218 void *PREFIX(valloc)(ZONE_ARG size_t size) __THROW
219 {
220 if (! memoryPageSize) initPageSize();
221
222 return scalable_aligned_malloc(size, memoryPageSize);
223 }
224
225 #undef ZONE_ARG
226 #undef PREFIX
227
228 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED
229
230 // match prototype from system headers
231 #if __ANDROID__
malloc_usable_size(const void * ptr)232 size_t malloc_usable_size(const void *ptr) __THROW
233 #else
234 size_t malloc_usable_size(void *ptr) __THROW
235 #endif
236 {
237 InitOrigPointers();
238 return __TBB_malloc_safer_msize(const_cast<void*>(ptr), (size_t (*)(void*))orig_msize);
239 }
240
posix_memalign(void ** memptr,size_t alignment,size_t size)241 int posix_memalign(void **memptr, size_t alignment, size_t size) __THROW
242 {
243 return scalable_posix_memalign(memptr, alignment, size);
244 }
245
246 /* pvalloc allocates smallest set of complete pages which can hold
247 the requested number of bytes. Result is aligned on page boundary. */
pvalloc(size_t size)248 void *pvalloc(size_t size) __THROW
249 {
250 if (! memoryPageSize) initPageSize();
251 // align size up to the page size,
252 // pvalloc(0) returns 1 page, see man libmpatrol
253 size = size? ((size-1) | (memoryPageSize-1)) + 1 : memoryPageSize;
254
255 return scalable_aligned_malloc(size, memoryPageSize);
256 }
257
mallopt(int,int)258 int mallopt(int /*param*/, int /*value*/) __THROW
259 {
260 return 1;
261 }
262
263 #if defined(__GLIBC__) || defined(__ANDROID__)
mallinfo()264 struct mallinfo mallinfo() __THROW
265 {
266 struct mallinfo m;
267 memset(&m, 0, sizeof(struct mallinfo));
268
269 return m;
270 }
271 #endif
272
273 #if __ANDROID__
274 // Android doesn't have malloc_usable_size, provide it to be compatible
275 // with Linux, in addition overload dlmalloc_usable_size() that presented
276 // under Android.
277 size_t dlmalloc_usable_size(const void *ptr) __TBB_ALIAS_ATTR_COPY(malloc_usable_size);
278 #else // __ANDROID__
279 // TODO: consider using __typeof__ to guarantee the correct declaration types
280 // C11 function, supported starting GLIBC 2.16
281 void *aligned_alloc(size_t alignment, size_t size) __TBB_ALIAS_ATTR_COPY(memalign);
282 // Those non-standard functions are exported by GLIBC, and might be used
283 // in conjunction with standard malloc/free, so we must overload them.
284 // Bionic doesn't have them. Not removing from the linker scripts,
285 // as absent entry points are ignored by the linker.
286
287 void *__libc_malloc(size_t size) __TBB_ALIAS_ATTR_COPY(malloc);
288 void *__libc_calloc(size_t num, size_t size) __TBB_ALIAS_ATTR_COPY(calloc);
289 void *__libc_memalign(size_t alignment, size_t size) __TBB_ALIAS_ATTR_COPY(memalign);
290 void *__libc_pvalloc(size_t size) __TBB_ALIAS_ATTR_COPY(pvalloc);
291 void *__libc_valloc(size_t size) __TBB_ALIAS_ATTR_COPY(valloc);
292
293 // call original __libc_* to support naive replacement of free via __libc_free etc
__libc_free(void * ptr)294 void __libc_free(void *ptr)
295 {
296 InitOrigPointers();
297 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_libc_free);
298 }
299
__libc_realloc(void * ptr,size_t size)300 void *__libc_realloc(void *ptr, size_t size)
301 {
302 InitOrigPointers();
303 return __TBB_malloc_safer_realloc(ptr, size, orig_libc_realloc);
304 }
305 #endif // !__ANDROID__
306
307 } /* extern "C" */
308
309 /*** replacements for global operators new and delete ***/
310
operator new(size_t sz)311 void* operator new(size_t sz) {
312 return InternalOperatorNew(sz);
313 }
operator new[](size_t sz)314 void* operator new[](size_t sz) {
315 return InternalOperatorNew(sz);
316 }
operator delete(void * ptr)317 void operator delete(void* ptr) noexcept {
318 InitOrigPointers();
319 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);
320 }
operator delete[](void * ptr)321 void operator delete[](void* ptr) noexcept {
322 InitOrigPointers();
323 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);
324 }
operator new(size_t sz,const std::nothrow_t &)325 void* operator new(size_t sz, const std::nothrow_t&) noexcept {
326 return scalable_malloc(sz);
327 }
operator new[](std::size_t sz,const std::nothrow_t &)328 void* operator new[](std::size_t sz, const std::nothrow_t&) noexcept {
329 return scalable_malloc(sz);
330 }
operator delete(void * ptr,const std::nothrow_t &)331 void operator delete(void* ptr, const std::nothrow_t&) noexcept {
332 InitOrigPointers();
333 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);
334 }
operator delete[](void * ptr,const std::nothrow_t &)335 void operator delete[](void* ptr, const std::nothrow_t&) noexcept {
336 InitOrigPointers();
337 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);
338 }
339
340 #endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED */
341 #endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED */
342
343 #ifdef _WIN32
344 #include <windows.h>
345
346 #if !__TBB_WIN8UI_SUPPORT
347
348 #include <stdio.h>
349
350 #include "function_replacement.h"
351
352 template<typename T, size_t N> // generic function to find length of array
arrayLength(const T (&)[N])353 inline size_t arrayLength(const T(&)[N]) {
354 return N;
355 }
356
__TBB_malloc_safer_delete(void * ptr)357 void __TBB_malloc_safer_delete( void *ptr)
358 {
359 __TBB_malloc_safer_free( ptr, nullptr );
360 }
361
safer_aligned_malloc(size_t size,size_t alignment)362 void* safer_aligned_malloc( size_t size, size_t alignment )
363 {
364 // workaround for "is power of 2 pow N" bug that accepts zeros
365 return scalable_aligned_malloc( size, alignment>sizeof(size_t*)?alignment:sizeof(size_t*) );
366 }
367
368 // we do not support _expand();
safer_expand(void *,size_t)369 void* safer_expand( void *, size_t )
370 {
371 return nullptr;
372 }
373
374 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(CRTLIB) \
375 void (*orig_free_##CRTLIB)(void*); \
376 void __TBB_malloc_safer_free_##CRTLIB(void *ptr) \
377 { \
378 __TBB_malloc_safer_free( ptr, orig_free_##CRTLIB ); \
379 } \
380 \
381 void (*orig__aligned_free_##CRTLIB)(void*); \
382 void __TBB_malloc_safer__aligned_free_##CRTLIB(void *ptr) \
383 { \
384 __TBB_malloc_safer_free( ptr, orig__aligned_free_##CRTLIB ); \
385 } \
386 \
387 size_t (*orig__msize_##CRTLIB)(void*); \
388 size_t __TBB_malloc_safer__msize_##CRTLIB(void *ptr) \
389 { \
390 return __TBB_malloc_safer_msize( ptr, orig__msize_##CRTLIB ); \
391 } \
392 \
393 size_t (*orig__aligned_msize_##CRTLIB)(void*, size_t, size_t); \
394 size_t __TBB_malloc_safer__aligned_msize_##CRTLIB( void *ptr, size_t alignment, size_t offset) \
395 { \
396 return __TBB_malloc_safer_aligned_msize( ptr, alignment, offset, orig__aligned_msize_##CRTLIB ); \
397 } \
398 \
399 void* __TBB_malloc_safer_realloc_##CRTLIB( void *ptr, size_t size ) \
400 { \
401 orig_ptrs func_ptrs = {orig_free_##CRTLIB, orig__msize_##CRTLIB}; \
402 return __TBB_malloc_safer_realloc( ptr, size, &func_ptrs ); \
403 } \
404 \
405 void* __TBB_malloc_safer__aligned_realloc_##CRTLIB( void *ptr, size_t size, size_t alignment ) \
406 { \
407 orig_aligned_ptrs func_ptrs = {orig__aligned_free_##CRTLIB, orig__aligned_msize_##CRTLIB}; \
408 return __TBB_malloc_safer_aligned_realloc( ptr, size, alignment, &func_ptrs ); \
409 }
410
411 // Only for ucrtbase: substitution for _o_free
412 void (*orig__o_free)(void*);
__TBB_malloc__o_free(void * ptr)413 void __TBB_malloc__o_free(void *ptr)
414 {
415 __TBB_malloc_safer_free( ptr, orig__o_free );
416 }
417 // Only for ucrtbase: substitution for _free_base
418 void(*orig__free_base)(void*);
__TBB_malloc__free_base(void * ptr)419 void __TBB_malloc__free_base(void *ptr)
420 {
421 __TBB_malloc_safer_free(ptr, orig__free_base);
422 }
423
424 // Size limit is MAX_PATTERN_SIZE (28) byte codes / 56 symbols per line.
425 // * can be used to match any digit in byte codes.
426 // # followed by several * indicate a relative address that needs to be corrected.
427 // Purpose of the pattern is to mark an instruction bound; it should consist of several
428 // full instructions plus one extra byte code. It's not required for the patterns
429 // to be unique (i.e., it's OK to have same pattern for unrelated functions).
430 // TODO: use hot patch prologues if exist
431 const char* known_bytecodes[] = {
432 #if _WIN64
433 // "========================================================" - 56 symbols
434 "4883EC284885C974", // release free()
435 "4883EC284885C975", // release _msize()
436 "4885C974375348", // release free() 8.0.50727.42, 10.0
437 "E907000000CCCC", // release _aligned_msize(), _aligned_free() ucrtbase.dll
438 "C7442410000000008B", // release free() ucrtbase.dll 10.0.14393.33
439 "E90B000000CCCC", // release _msize() ucrtbase.dll 10.0.14393.33
440 "48895C24085748", // release _aligned_msize() ucrtbase.dll 10.0.14393.33
441 "E903000000CCCC", // release _aligned_msize() ucrtbase.dll 10.0.16299.522
442 "48894C24084883EC28BA", // debug prologue
443 "4C894424184889542410", // debug _aligned_msize() 10.0
444 "48894C24084883EC2848", // debug _aligned_free 10.0
445 "488BD1488D0D#*******E9", // _o_free(), ucrtbase.dll
446 #if __TBB_OVERLOAD_OLD_MSVCR
447 "48895C2408574883EC3049", // release _aligned_msize 9.0
448 "4883EC384885C975", // release _msize() 9.0
449 "4C8BC1488B0DA6E4040033", // an old win64 SDK
450 #endif
451 #else // _WIN32
452 // "========================================================" - 56 symbols
453 "8BFF558BEC8B", // multiple
454 "8BFF558BEC83", // release free() & _msize() 10.0.40219.325, _msize() ucrtbase.dll
455 "8BFF558BECFF", // release _aligned_msize ucrtbase.dll
456 "8BFF558BEC51", // release free() & _msize() ucrtbase.dll 10.0.14393.33
457 "558BEC8B450885C074", // release _aligned_free 11.0
458 "558BEC837D08000F", // release _msize() 11.0.51106.1
459 "558BEC837D08007419FF", // release free() 11.0.50727.1
460 "558BEC8B450885C075", // release _aligned_msize() 11.0.50727.1
461 "558BEC6A018B", // debug free() & _msize() 11.0
462 "558BEC8B451050", // debug _aligned_msize() 11.0
463 "558BEC8B450850", // debug _aligned_free 11.0
464 "8BFF558BEC6A", // debug free() & _msize() 10.0.40219.325
465 #if __TBB_OVERLOAD_OLD_MSVCR
466 "6A1868********E8", // release free() 8.0.50727.4053, 9.0
467 "6A1C68********E8", // release _msize() 8.0.50727.4053, 9.0
468 #endif
469 #endif // _WIN64/_WIN32
470 nullptr
471 };
472
473 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,function_name,dbgsuffix) \
474 ReplaceFunctionWithStore( #CRT_VER #dbgsuffix ".dll", #function_name, \
475 (FUNCPTR)__TBB_malloc_safer_##function_name##_##CRT_VER##dbgsuffix, \
476 known_bytecodes, (FUNCPTR*)&orig_##function_name##_##CRT_VER##dbgsuffix );
477
478 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,function_name,dbgsuffix) \
479 ReplaceFunctionWithStore( #CRT_VER #dbgsuffix ".dll", #function_name, \
480 (FUNCPTR)__TBB_malloc_safer_##function_name##_##CRT_VER##dbgsuffix, 0, nullptr );
481
482 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_REDIRECT(CRT_VER,function_name,dest_func,dbgsuffix) \
483 ReplaceFunctionWithStore( #CRT_VER #dbgsuffix ".dll", #function_name, \
484 (FUNCPTR)__TBB_malloc_safer_##dest_func##_##CRT_VER##dbgsuffix, 0, nullptr );
485
486 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,dbgsuffix) \
487 if (BytecodesAreKnown(#CRT_VER #dbgsuffix ".dll")) { \
488 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,free,dbgsuffix) \
489 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,_msize,dbgsuffix) \
490 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,realloc,dbgsuffix) \
491 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,_aligned_free,dbgsuffix) \
492 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,_aligned_msize,dbgsuffix) \
493 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,_aligned_realloc,dbgsuffix) \
494 } else \
495 SkipReplacement(#CRT_VER #dbgsuffix ".dll");
496
497 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_RELEASE(CRT_VER) __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,)
498 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_DEBUG(CRT_VER) __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,d)
499
500 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(CRT_VER) \
501 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_RELEASE(CRT_VER) \
502 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_DEBUG(CRT_VER)
503
504 #if __TBB_OVERLOAD_OLD_MSVCR
505 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr70d);
506 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr70);
507 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr71d);
508 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr71);
509 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr80d);
510 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr80);
511 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr90d);
512 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr90);
513 #endif
514 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr100d);
515 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr100);
516 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr110d);
517 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr110);
518 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120d);
519 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120);
520 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(ucrtbase);
521
522 /*** replacements for global operators new and delete ***/
523
524 #if _MSC_VER && !defined(__INTEL_COMPILER)
525 #pragma warning( push )
526 #pragma warning( disable : 4290 )
527 #endif
528
529 /*** operator new overloads internals (Linux, Windows) ***/
530
operator_new(size_t sz)531 void* operator_new(size_t sz) {
532 return InternalOperatorNew(sz);
533 }
operator_new_arr(size_t sz)534 void* operator_new_arr(size_t sz) {
535 return InternalOperatorNew(sz);
536 }
operator_delete(void * ptr)537 void operator_delete(void* ptr) noexcept {
538 __TBB_malloc_safer_delete(ptr);
539 }
540 #if _MSC_VER && !defined(__INTEL_COMPILER)
541 #pragma warning( pop )
542 #endif
543
operator_delete_arr(void * ptr)544 void operator_delete_arr(void* ptr) noexcept {
545 __TBB_malloc_safer_delete(ptr);
546 }
operator_new_t(size_t sz,const std::nothrow_t &)547 void* operator_new_t(size_t sz, const std::nothrow_t&) noexcept {
548 return scalable_malloc(sz);
549 }
operator_new_arr_t(std::size_t sz,const std::nothrow_t &)550 void* operator_new_arr_t(std::size_t sz, const std::nothrow_t&) noexcept {
551 return scalable_malloc(sz);
552 }
operator_delete_t(void * ptr,const std::nothrow_t &)553 void operator_delete_t(void* ptr, const std::nothrow_t&) noexcept {
554 __TBB_malloc_safer_delete(ptr);
555 }
operator_delete_arr_t(void * ptr,const std::nothrow_t &)556 void operator_delete_arr_t(void* ptr, const std::nothrow_t&) noexcept {
557 __TBB_malloc_safer_delete(ptr);
558 }
559
560 struct Module {
561 const char *name;
562 bool doFuncReplacement; // do replacement in the DLL
563 };
564
565 Module modules_to_replace[] = {
566 {"msvcr100d.dll", true},
567 {"msvcr100.dll", true},
568 {"msvcr110d.dll", true},
569 {"msvcr110.dll", true},
570 {"msvcr120d.dll", true},
571 {"msvcr120.dll", true},
572 {"ucrtbase.dll", true},
573 // "ucrtbased.dll" is not supported because of problems with _dbg functions
574 #if __TBB_OVERLOAD_OLD_MSVCR
575 {"msvcr90d.dll", true},
576 {"msvcr90.dll", true},
577 {"msvcr80d.dll", true},
578 {"msvcr80.dll", true},
579 {"msvcr70d.dll", true},
580 {"msvcr70.dll", true},
581 {"msvcr71d.dll", true},
582 {"msvcr71.dll", true},
583 #endif
584 #if __TBB_TODO
585 // TODO: Try enabling replacement for non-versioned system binaries below
586 {"msvcrtd.dll", true},
587 {"msvcrt.dll", true},
588 #endif
589 };
590
591 /*
592 We need to replace following functions:
593 malloc
594 calloc
595 _aligned_malloc
596 _expand (by dummy implementation)
597 ??2@YAPAXI@Z operator new (ia32)
598 ??_U@YAPAXI@Z void * operator new[] (size_t size) (ia32)
599 ??3@YAXPAX@Z operator delete (ia32)
600 ??_V@YAXPAX@Z operator delete[] (ia32)
601 ??2@YAPEAX_K@Z void * operator new(unsigned __int64) (intel64)
602 ??_V@YAXPEAX@Z void * operator new[](unsigned __int64) (intel64)
603 ??3@YAXPEAX@Z operator delete (intel64)
604 ??_V@YAXPEAX@Z operator delete[] (intel64)
605 ??2@YAPAXIABUnothrow_t@std@@@Z void * operator new (size_t sz, const std::nothrow_t&) throw() (optional)
606 ??_U@YAPAXIABUnothrow_t@std@@@Z void * operator new[] (size_t sz, const std::nothrow_t&) throw() (optional)
607
608 and these functions have runtime-specific replacement:
609 realloc
610 free
611 _msize
612 _aligned_realloc
613 _aligned_free
614 _aligned_msize
615 */
616
617 typedef struct FRData_t {
618 //char *_module;
619 const char *_func;
620 FUNCPTR _fptr;
621 FRR_ON_ERROR _on_error;
622 } FRDATA;
623
624 FRDATA c_routines_to_replace[] = {
625 { "malloc", (FUNCPTR)scalable_malloc, FRR_FAIL },
626 { "calloc", (FUNCPTR)scalable_calloc, FRR_FAIL },
627 { "_aligned_malloc", (FUNCPTR)safer_aligned_malloc, FRR_FAIL },
628 { "_expand", (FUNCPTR)safer_expand, FRR_IGNORE },
629 };
630
631 FRDATA cxx_routines_to_replace[] = {
632 #if _WIN64
633 { "??2@YAPEAX_K@Z", (FUNCPTR)operator_new, FRR_FAIL },
634 { "??_U@YAPEAX_K@Z", (FUNCPTR)operator_new_arr, FRR_FAIL },
635 { "??3@YAXPEAX@Z", (FUNCPTR)operator_delete, FRR_FAIL },
636 { "??_V@YAXPEAX@Z", (FUNCPTR)operator_delete_arr, FRR_FAIL },
637 #else
638 { "??2@YAPAXI@Z", (FUNCPTR)operator_new, FRR_FAIL },
639 { "??_U@YAPAXI@Z", (FUNCPTR)operator_new_arr, FRR_FAIL },
640 { "??3@YAXPAX@Z", (FUNCPTR)operator_delete, FRR_FAIL },
641 { "??_V@YAXPAX@Z", (FUNCPTR)operator_delete_arr, FRR_FAIL },
642 #endif
643 { "??2@YAPAXIABUnothrow_t@std@@@Z", (FUNCPTR)operator_new_t, FRR_IGNORE },
644 { "??_U@YAPAXIABUnothrow_t@std@@@Z", (FUNCPTR)operator_new_arr_t, FRR_IGNORE }
645 };
646
647 #ifndef UNICODE
648 typedef char unicode_char_t;
649 #define WCHAR_SPEC "%s"
650 #else
651 typedef wchar_t unicode_char_t;
652 #define WCHAR_SPEC "%ls"
653 #endif
654
655 // Check that we recognize bytecodes that should be replaced by trampolines.
656 // If some functions have unknown prologue patterns, replacement should not be done.
BytecodesAreKnown(const unicode_char_t * dllName)657 bool BytecodesAreKnown(const unicode_char_t *dllName)
658 {
659 const char *funcName[] = {"free", "_msize", "_aligned_free", "_aligned_msize", 0};
660 HMODULE module = GetModuleHandle(dllName);
661
662 if (!module)
663 return false;
664 for (int i=0; funcName[i]; i++)
665 if (! IsPrologueKnown(dllName, funcName[i], known_bytecodes, module)) {
666 fprintf(stderr, "TBBmalloc: skip allocation functions replacement in " WCHAR_SPEC
667 ": unknown prologue for function " WCHAR_SPEC "\n", dllName, funcName[i]);
668 return false;
669 }
670 return true;
671 }
672
SkipReplacement(const unicode_char_t * dllName)673 void SkipReplacement(const unicode_char_t *dllName)
674 {
675 #ifndef UNICODE
676 const char *dllStr = dllName;
677 #else
678 const size_t sz = 128; // all DLL name must fit
679
680 char buffer[sz];
681 size_t real_sz;
682 char *dllStr = buffer;
683
684 errno_t ret = wcstombs_s(&real_sz, dllStr, sz, dllName, sz-1);
685 __TBB_ASSERT(!ret, "Dll name conversion failed");
686 #endif
687
688 for (size_t i=0; i<arrayLength(modules_to_replace); i++)
689 if (!strcmp(modules_to_replace[i].name, dllStr)) {
690 modules_to_replace[i].doFuncReplacement = false;
691 break;
692 }
693 }
694
ReplaceFunctionWithStore(const unicode_char_t * dllName,const char * funcName,FUNCPTR newFunc,const char ** opcodes,FUNCPTR * origFunc,FRR_ON_ERROR on_error=FRR_FAIL)695 void ReplaceFunctionWithStore( const unicode_char_t *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc, FRR_ON_ERROR on_error = FRR_FAIL )
696 {
697 FRR_TYPE res = ReplaceFunction( dllName, funcName, newFunc, opcodes, origFunc );
698
699 if (res == FRR_OK || res == FRR_NODLL || (res == FRR_NOFUNC && on_error == FRR_IGNORE))
700 return;
701
702 fprintf(stderr, "Failed to %s function %s in module %s\n",
703 res==FRR_NOFUNC? "find" : "replace", funcName, dllName);
704
705 // Unable to replace a required function
706 // Aborting because incomplete replacement of memory management functions
707 // may leave the program in an invalid state
708 abort();
709 }
710
doMallocReplacement()711 void doMallocReplacement()
712 {
713 // Replace functions and keep backup of original code (separate for each runtime)
714 #if __TBB_OVERLOAD_OLD_MSVCR
715 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr70)
716 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr71)
717 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr80)
718 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr90)
719 #endif
720 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr100)
721 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr110)
722 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr120)
723 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_RELEASE(ucrtbase)
724
725 // Replace functions without storing original code
726 for (size_t j = 0; j < arrayLength(modules_to_replace); j++) {
727 if (!modules_to_replace[j].doFuncReplacement)
728 continue;
729 for (size_t i = 0; i < arrayLength(c_routines_to_replace); i++)
730 {
731 ReplaceFunctionWithStore( modules_to_replace[j].name, c_routines_to_replace[i]._func, c_routines_to_replace[i]._fptr, nullptr, nullptr, c_routines_to_replace[i]._on_error );
732 }
733 if ( strcmp(modules_to_replace[j].name, "ucrtbase.dll") == 0 ) {
734 HMODULE ucrtbase_handle = GetModuleHandle("ucrtbase.dll");
735 if (!ucrtbase_handle)
736 continue;
737 // If _o_free function is present and patchable, redirect it to tbbmalloc as well
738 // This prevents issues with other _o_* functions which might allocate memory with malloc
739 if ( IsPrologueKnown("ucrtbase.dll", "_o_free", known_bytecodes, ucrtbase_handle)) {
740 ReplaceFunctionWithStore( "ucrtbase.dll", "_o_free", (FUNCPTR)__TBB_malloc__o_free, known_bytecodes, (FUNCPTR*)&orig__o_free, FRR_FAIL );
741 }
742 // Similarly for _free_base
743 if (IsPrologueKnown("ucrtbase.dll", "_free_base", known_bytecodes, ucrtbase_handle)) {
744 ReplaceFunctionWithStore("ucrtbase.dll", "_free_base", (FUNCPTR)__TBB_malloc__free_base, known_bytecodes, (FUNCPTR*)&orig__free_base, FRR_FAIL);
745 }
746 // ucrtbase.dll does not export operator new/delete, so skip the rest of the loop.
747 continue;
748 }
749
750 for (size_t i = 0; i < arrayLength(cxx_routines_to_replace); i++)
751 {
752 #if !_WIN64
753 // in Microsoft* Visual Studio* 2012 and 2013 32-bit operator delete consists of 2 bytes only: short jump to free(ptr);
754 // replacement should be skipped for this particular case.
755 if ( ((strcmp(modules_to_replace[j].name, "msvcr110.dll") == 0) || (strcmp(modules_to_replace[j].name, "msvcr120.dll") == 0)) && (strcmp(cxx_routines_to_replace[i]._func, "??3@YAXPAX@Z") == 0) ) continue;
756 // in Microsoft* Visual Studio* 2013 32-bit operator delete[] consists of 2 bytes only: short jump to free(ptr);
757 // replacement should be skipped for this particular case.
758 if ( (strcmp(modules_to_replace[j].name, "msvcr120.dll") == 0) && (strcmp(cxx_routines_to_replace[i]._func, "??_V@YAXPAX@Z") == 0) ) continue;
759 #endif
760 ReplaceFunctionWithStore( modules_to_replace[j].name, cxx_routines_to_replace[i]._func, cxx_routines_to_replace[i]._fptr, nullptr, nullptr, cxx_routines_to_replace[i]._on_error );
761 }
762 }
763 }
764
765 #endif // !__TBB_WIN8UI_SUPPORT
766
767 #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
768 // Suppress warning for UWP build ('main' signature found without threading model)
769 #pragma warning(push)
770 #pragma warning(disable:4447)
771 #endif
772
DllMain(HINSTANCE hInst,DWORD callReason,LPVOID reserved)773 extern "C" BOOL WINAPI DllMain( HINSTANCE hInst, DWORD callReason, LPVOID reserved )
774 {
775
776 if ( callReason==DLL_PROCESS_ATTACH && reserved && hInst ) {
777 #if !__TBB_WIN8UI_SUPPORT
778 if (!tbb::detail::r1::GetBoolEnvironmentVariable("TBB_MALLOC_DISABLE_REPLACEMENT"))
779 {
780 doMallocReplacement();
781 }
782 #endif // !__TBB_WIN8UI_SUPPORT
783 }
784
785 return TRUE;
786 }
787
788 #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
789 #pragma warning(pop)
790 #endif
791
792 // Just to make the linker happy and link the DLL to the application
__TBB_malloc_proxy()793 extern "C" __declspec(dllexport) void __TBB_malloc_proxy()
794 {
795
796 }
797
798 #endif //_WIN32
799