1 //===-- memprof_malloc_linux.cpp -----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProfiler, a memory profiler.
10 //
11 // Linux-specific malloc interception.
12 // We simply define functions like malloc, free, realloc, etc.
13 // They will replace the corresponding libc functions automagically.
14 //===----------------------------------------------------------------------===//
15 
16 #include "sanitizer_common/sanitizer_platform.h"
17 #if !SANITIZER_LINUX
18 #error Unsupported OS
19 #endif
20 
21 #include "memprof_allocator.h"
22 #include "memprof_interceptors.h"
23 #include "memprof_internal.h"
24 #include "memprof_stack.h"
25 #include "sanitizer_common/sanitizer_allocator_checks.h"
26 #include "sanitizer_common/sanitizer_errno.h"
27 #include "sanitizer_common/sanitizer_tls_get_addr.h"
28 
29 // ---------------------- Replacement functions ---------------- {{{1
30 using namespace __memprof;
31 
32 static uptr allocated_for_dlsym;
33 static uptr last_dlsym_alloc_size_in_words;
34 static const uptr kDlsymAllocPoolSize = 1024;
35 static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
36 
37 static inline bool IsInDlsymAllocPool(const void *ptr) {
38   uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
39   return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
40 }
41 
42 static void *AllocateFromLocalPool(uptr size_in_bytes) {
43   uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
44   void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
45   last_dlsym_alloc_size_in_words = size_in_words;
46   allocated_for_dlsym += size_in_words;
47   CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
48   return mem;
49 }
50 
51 static void DeallocateFromLocalPool(const void *ptr) {
52   // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
53   // error messages and instead uses malloc followed by free. To avoid pool
54   // exhaustion due to long object filenames, handle that special case here.
55   uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
56   void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset];
57   if (prev_mem == ptr) {
58     REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
59     allocated_for_dlsym = prev_offset;
60     last_dlsym_alloc_size_in_words = 0;
61   }
62 }
63 
64 static inline bool MaybeInDlsym() { return memprof_init_is_running; }
65 
66 static inline bool UseLocalPool() { return MaybeInDlsym(); }
67 
68 static void *ReallocFromLocalPool(void *ptr, uptr size) {
69   const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
70   const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
71   void *new_ptr;
72   if (UNLIKELY(UseLocalPool())) {
73     new_ptr = AllocateFromLocalPool(size);
74   } else {
75     ENSURE_MEMPROF_INITED();
76     GET_STACK_TRACE_MALLOC;
77     new_ptr = memprof_malloc(size, &stack);
78   }
79   internal_memcpy(new_ptr, ptr, copy_size);
80   return new_ptr;
81 }
82 
83 INTERCEPTOR(void, free, void *ptr) {
84   if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
85     DeallocateFromLocalPool(ptr);
86     return;
87   }
88   GET_STACK_TRACE_FREE;
89   memprof_free(ptr, &stack, FROM_MALLOC);
90 }
91 
92 #if SANITIZER_INTERCEPT_CFREE
93 INTERCEPTOR(void, cfree, void *ptr) {
94   if (UNLIKELY(IsInDlsymAllocPool(ptr)))
95     return;
96   GET_STACK_TRACE_FREE;
97   memprof_free(ptr, &stack, FROM_MALLOC);
98 }
99 #endif // SANITIZER_INTERCEPT_CFREE
100 
101 INTERCEPTOR(void *, malloc, uptr size) {
102   if (UNLIKELY(UseLocalPool()))
103     // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
104     return AllocateFromLocalPool(size);
105   ENSURE_MEMPROF_INITED();
106   GET_STACK_TRACE_MALLOC;
107   return memprof_malloc(size, &stack);
108 }
109 
110 INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) {
111   if (UNLIKELY(UseLocalPool()))
112     // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
113     return AllocateFromLocalPool(nmemb * size);
114   ENSURE_MEMPROF_INITED();
115   GET_STACK_TRACE_MALLOC;
116   return memprof_calloc(nmemb, size, &stack);
117 }
118 
119 INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
120   if (UNLIKELY(IsInDlsymAllocPool(ptr)))
121     return ReallocFromLocalPool(ptr, size);
122   if (UNLIKELY(UseLocalPool()))
123     return AllocateFromLocalPool(size);
124   ENSURE_MEMPROF_INITED();
125   GET_STACK_TRACE_MALLOC;
126   return memprof_realloc(ptr, size, &stack);
127 }
128 
129 #if SANITIZER_INTERCEPT_REALLOCARRAY
130 INTERCEPTOR(void *, reallocarray, void *ptr, uptr nmemb, uptr size) {
131   ENSURE_MEMPROF_INITED();
132   GET_STACK_TRACE_MALLOC;
133   return memprof_reallocarray(ptr, nmemb, size, &stack);
134 }
135 #endif // SANITIZER_INTERCEPT_REALLOCARRAY
136 
137 #if SANITIZER_INTERCEPT_MEMALIGN
138 INTERCEPTOR(void *, memalign, uptr boundary, uptr size) {
139   GET_STACK_TRACE_MALLOC;
140   return memprof_memalign(boundary, size, &stack, FROM_MALLOC);
141 }
142 
143 INTERCEPTOR(void *, __libc_memalign, uptr boundary, uptr size) {
144   GET_STACK_TRACE_MALLOC;
145   void *res = memprof_memalign(boundary, size, &stack, FROM_MALLOC);
146   DTLS_on_libc_memalign(res, size);
147   return res;
148 }
149 #endif // SANITIZER_INTERCEPT_MEMALIGN
150 
151 #if SANITIZER_INTERCEPT_ALIGNED_ALLOC
152 INTERCEPTOR(void *, aligned_alloc, uptr boundary, uptr size) {
153   GET_STACK_TRACE_MALLOC;
154   return memprof_aligned_alloc(boundary, size, &stack);
155 }
156 #endif // SANITIZER_INTERCEPT_ALIGNED_ALLOC
157 
158 INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
159   GET_CURRENT_PC_BP_SP;
160   (void)sp;
161   return memprof_malloc_usable_size(ptr, pc, bp);
162 }
163 
164 #if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
165 // We avoid including malloc.h for portability reasons.
166 // man mallinfo says the fields are "long", but the implementation uses int.
167 // It doesn't matter much -- we just need to make sure that the libc's mallinfo
168 // is not called.
169 struct fake_mallinfo {
170   int x[10];
171 };
172 
173 INTERCEPTOR(struct fake_mallinfo, mallinfo, void) {
174   struct fake_mallinfo res;
175   REAL(memset)(&res, 0, sizeof(res));
176   return res;
177 }
178 
179 INTERCEPTOR(int, mallopt, int cmd, int value) { return 0; }
180 #endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
181 
182 INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
183   GET_STACK_TRACE_MALLOC;
184   return memprof_posix_memalign(memptr, alignment, size, &stack);
185 }
186 
187 INTERCEPTOR(void *, valloc, uptr size) {
188   GET_STACK_TRACE_MALLOC;
189   return memprof_valloc(size, &stack);
190 }
191 
192 #if SANITIZER_INTERCEPT_PVALLOC
193 INTERCEPTOR(void *, pvalloc, uptr size) {
194   GET_STACK_TRACE_MALLOC;
195   return memprof_pvalloc(size, &stack);
196 }
197 #endif // SANITIZER_INTERCEPT_PVALLOC
198 
199 INTERCEPTOR(void, malloc_stats, void) { __memprof_print_accumulated_stats(); }
200 
201 namespace __memprof {
202 void ReplaceSystemMalloc() {}
203 } // namespace __memprof
204