1 //===-- memprof_rtl.cpp --------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProfiler, a memory profiler.
10 //
11 // Main file of the MemProf run-time library.
12 //===----------------------------------------------------------------------===//
13 
14 #include "memprof_allocator.h"
15 #include "memprof_interceptors.h"
16 #include "memprof_interface_internal.h"
17 #include "memprof_internal.h"
18 #include "memprof_mapping.h"
19 #include "memprof_stack.h"
20 #include "memprof_stats.h"
21 #include "memprof_thread.h"
22 #include "sanitizer_common/sanitizer_atomic.h"
23 #include "sanitizer_common/sanitizer_flags.h"
24 #include "sanitizer_common/sanitizer_libc.h"
25 #include "sanitizer_common/sanitizer_symbolizer.h"
26 #include <ctime>
27 
28 uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol.
29 
30 namespace __memprof {
31 
32 static void MemprofDie() {
33   static atomic_uint32_t num_calls;
34   if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
35     // Don't die twice - run a busy loop.
36     while (1) {
37     }
38   }
39   if (common_flags()->print_module_map >= 1)
40     DumpProcessMap();
41   if (flags()->unmap_shadow_on_exit) {
42     if (kHighShadowEnd)
43       UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
44   }
45 }
46 
47 static void MemprofCheckFailed(const char *file, int line, const char *cond,
48                                u64 v1, u64 v2) {
49   Report("MemProfiler CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, line,
50          cond, (uptr)v1, (uptr)v2);
51 
52   // Print a stack trace the first time we come here. Otherwise, we probably
53   // failed a CHECK during symbolization.
54   static atomic_uint32_t num_calls;
55   if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) == 0) {
56     PRINT_CURRENT_STACK_CHECK();
57   }
58 
59   Die();
60 }
61 
62 // -------------------------- Globals --------------------- {{{1
63 int memprof_inited;
64 int memprof_init_done;
65 bool memprof_init_is_running;
66 int memprof_timestamp_inited;
67 long memprof_init_timestamp_s;
68 
69 uptr kHighMemEnd;
70 
71 // -------------------------- Run-time entry ------------------- {{{1
72 // exported functions
73 
74 #define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr);
75 
76 #define MEMPROF_MEMORY_ACCESS_CALLBACK(type)                                   \
77   extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) {   \
78     MEMPROF_MEMORY_ACCESS_CALLBACK_BODY()                                      \
79   }
80 
81 MEMPROF_MEMORY_ACCESS_CALLBACK(load)
82 MEMPROF_MEMORY_ACCESS_CALLBACK(store)
83 
84 // Force the linker to keep the symbols for various MemProf interface
85 // functions. We want to keep those in the executable in order to let the
86 // instrumented dynamic libraries access the symbol even if it is not used by
87 // the executable itself. This should help if the build system is removing dead
88 // code at link time.
89 static NOINLINE void force_interface_symbols() {
90   volatile int fake_condition = 0; // prevent dead condition elimination.
91   // clang-format off
92   switch (fake_condition) {
93     case 1: __memprof_record_access(nullptr); break;
94     case 2: __memprof_record_access_range(nullptr, 0); break;
95   }
96   // clang-format on
97 }
98 
99 static void memprof_atexit() {
100   Printf("MemProfiler exit stats:\n");
101   __memprof_print_accumulated_stats();
102 }
103 
104 static void InitializeHighMemEnd() {
105   kHighMemEnd = GetMaxUserVirtualAddress();
106   // Increase kHighMemEnd to make sure it's properly
107   // aligned together with kHighMemBeg:
108   kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
109 }
110 
111 void PrintAddressSpaceLayout() {
112   if (kHighMemBeg) {
113     Printf("|| `[%p, %p]` || HighMem    ||\n", (void *)kHighMemBeg,
114            (void *)kHighMemEnd);
115     Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg,
116            (void *)kHighShadowEnd);
117   }
118   Printf("|| `[%p, %p]` || ShadowGap  ||\n", (void *)kShadowGapBeg,
119          (void *)kShadowGapEnd);
120   if (kLowShadowBeg) {
121     Printf("|| `[%p, %p]` || LowShadow  ||\n", (void *)kLowShadowBeg,
122            (void *)kLowShadowEnd);
123     Printf("|| `[%p, %p]` || LowMem     ||\n", (void *)kLowMemBeg,
124            (void *)kLowMemEnd);
125   }
126   Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg),
127          (void *)MEM_TO_SHADOW(kLowShadowEnd));
128   if (kHighMemBeg) {
129     Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg),
130            (void *)MEM_TO_SHADOW(kHighShadowEnd));
131   }
132   Printf("\n");
133   Printf("malloc_context_size=%zu\n",
134          (uptr)common_flags()->malloc_context_size);
135 
136   Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
137   Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
138   Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
139   CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
140 }
141 
142 static bool UNUSED __local_memprof_dyninit = [] {
143   MaybeStartBackgroudThread();
144   SetSoftRssLimitExceededCallback(MemprofSoftRssLimitExceededCallback);
145 
146   return false;
147 }();
148 
149 static void MemprofInitInternal() {
150   if (LIKELY(memprof_inited))
151     return;
152   SanitizerToolName = "MemProfiler";
153   CHECK(!memprof_init_is_running && "MemProf init calls itself!");
154   memprof_init_is_running = true;
155 
156   CacheBinaryName();
157 
158   // Initialize flags. This must be done early, because most of the
159   // initialization steps look at flags().
160   InitializeFlags();
161 
162   AvoidCVE_2016_2143();
163 
164   SetMallocContextSize(common_flags()->malloc_context_size);
165 
166   InitializeHighMemEnd();
167 
168   // Make sure we are not statically linked.
169   MemprofDoesNotSupportStaticLinkage();
170 
171   // Install tool-specific callbacks in sanitizer_common.
172   AddDieCallback(MemprofDie);
173   SetCheckFailedCallback(MemprofCheckFailed);
174 
175   __sanitizer_set_report_path(common_flags()->log_path);
176 
177   __sanitizer::InitializePlatformEarly();
178 
179   // Re-exec ourselves if we need to set additional env or command line args.
180   MaybeReexec();
181 
182   // Setup internal allocator callback.
183   SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
184 
185   InitializeMemprofInterceptors();
186   CheckASLR();
187 
188   ReplaceSystemMalloc();
189 
190   DisableCoreDumperIfNecessary();
191 
192   InitializeShadowMemory();
193 
194   TSDInit(PlatformTSDDtor);
195 
196   InitializeAllocator();
197 
198   // On Linux MemprofThread::ThreadStart() calls malloc() that's why
199   // memprof_inited should be set to 1 prior to initializing the threads.
200   memprof_inited = 1;
201   memprof_init_is_running = false;
202 
203   if (flags()->atexit)
204     Atexit(memprof_atexit);
205 
206   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
207 
208   // interceptors
209   InitTlsSize();
210 
211   // Create main thread.
212   MemprofThread *main_thread = CreateMainThread();
213   CHECK_EQ(0, main_thread->tid());
214   force_interface_symbols(); // no-op.
215   SanitizerInitializeUnwinder();
216 
217   Symbolizer::LateInitialize();
218 
219   VReport(1, "MemProfiler Init done\n");
220 
221   memprof_init_done = 1;
222 }
223 
224 void MemprofInitTime() {
225   if (LIKELY(memprof_timestamp_inited))
226     return;
227   timespec ts;
228   timespec_get(&ts, TIME_UTC);
229   memprof_init_timestamp_s = ts.tv_sec;
230   memprof_timestamp_inited = 1;
231 }
232 
233 // Initialize as requested from some part of MemProf runtime library
234 // (interceptors, allocator, etc).
235 void MemprofInitFromRtl() { MemprofInitInternal(); }
236 
237 #if MEMPROF_DYNAMIC
238 // Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable
239 // (and thus normal initializers from .preinit_array or modules haven't run).
240 
241 class MemprofInitializer {
242 public:
243   MemprofInitializer() { MemprofInitFromRtl(); }
244 };
245 
246 static MemprofInitializer memprof_initializer;
247 #endif // MEMPROF_DYNAMIC
248 
249 } // namespace __memprof
250 
251 // ---------------------- Interface ---------------- {{{1
252 using namespace __memprof;
253 
254 // Initialize as requested from instrumented application code.
255 void __memprof_init() {
256   MemprofInitTime();
257   MemprofInitInternal();
258 }
259 
260 void __memprof_preinit() { MemprofInitInternal(); }
261 
262 void __memprof_version_mismatch_check_v1() {}
263 
264 void __memprof_record_access(void const volatile *addr) {
265   __memprof::RecordAccess((uptr)addr);
266 }
267 
268 // We only record the access on the first location in the range,
269 // since we will later accumulate the access counts across the
270 // full allocation, and we don't want to inflate the hotness from
271 // a memory intrinsic on a large range of memory.
272 // TODO: Should we do something else so we can better track utilization?
273 void __memprof_record_access_range(void const volatile *addr,
274                                    UNUSED uptr size) {
275   __memprof::RecordAccess((uptr)addr);
276 }
277 
278 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16
279 __sanitizer_unaligned_load16(const uu16 *p) {
280   __memprof_record_access(p);
281   return *p;
282 }
283 
284 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32
285 __sanitizer_unaligned_load32(const uu32 *p) {
286   __memprof_record_access(p);
287   return *p;
288 }
289 
290 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
291 __sanitizer_unaligned_load64(const uu64 *p) {
292   __memprof_record_access(p);
293   return *p;
294 }
295 
296 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
297 __sanitizer_unaligned_store16(uu16 *p, u16 x) {
298   __memprof_record_access(p);
299   *p = x;
300 }
301 
302 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
303 __sanitizer_unaligned_store32(uu32 *p, u32 x) {
304   __memprof_record_access(p);
305   *p = x;
306 }
307 
308 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
309 __sanitizer_unaligned_store64(uu64 *p, u64 x) {
310   __memprof_record_access(p);
311   *p = x;
312 }
313