1 //===-- memprof_rtl.cpp --------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProfiler, a memory profiler.
10 //
11 // Main file of the MemProf run-time library.
12 //===----------------------------------------------------------------------===//
13 
14 #include "memprof_allocator.h"
15 #include "memprof_interceptors.h"
16 #include "memprof_interface_internal.h"
17 #include "memprof_internal.h"
18 #include "memprof_mapping.h"
19 #include "memprof_stack.h"
20 #include "memprof_stats.h"
21 #include "memprof_thread.h"
22 #include "sanitizer_common/sanitizer_atomic.h"
23 #include "sanitizer_common/sanitizer_flags.h"
24 #include "sanitizer_common/sanitizer_libc.h"
25 #include "sanitizer_common/sanitizer_symbolizer.h"
26 #include <ctime>
27 
28 uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol.
29 
30 // Allow the user to specify a profile output file via the binary.
31 SANITIZER_WEAK_ATTRIBUTE char __memprof_profile_filename[1];
32 
33 namespace __memprof {
34 
35 static void MemprofDie() {
36   static atomic_uint32_t num_calls;
37   if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
38     // Don't die twice - run a busy loop.
39     while (1) {
40     }
41   }
42   if (common_flags()->print_module_map >= 1)
43     DumpProcessMap();
44   if (flags()->unmap_shadow_on_exit) {
45     if (kHighShadowEnd)
46       UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
47   }
48 }
49 
50 static void MemprofCheckFailed(const char *file, int line, const char *cond,
51                                u64 v1, u64 v2) {
52   Report("MemProfiler CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, line,
53          cond, (uptr)v1, (uptr)v2);
54 
55   // Print a stack trace the first time we come here. Otherwise, we probably
56   // failed a CHECK during symbolization.
57   static atomic_uint32_t num_calls;
58   if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) == 0) {
59     PRINT_CURRENT_STACK_CHECK();
60   }
61 
62   Die();
63 }
64 
65 // -------------------------- Globals --------------------- {{{1
66 int memprof_inited;
67 int memprof_init_done;
68 bool memprof_init_is_running;
69 int memprof_timestamp_inited;
70 long memprof_init_timestamp_s;
71 
72 uptr kHighMemEnd;
73 
74 // -------------------------- Run-time entry ------------------- {{{1
75 // exported functions
76 
77 #define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr);
78 
79 #define MEMPROF_MEMORY_ACCESS_CALLBACK(type)                                   \
80   extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) {   \
81     MEMPROF_MEMORY_ACCESS_CALLBACK_BODY()                                      \
82   }
83 
84 MEMPROF_MEMORY_ACCESS_CALLBACK(load)
85 MEMPROF_MEMORY_ACCESS_CALLBACK(store)
86 
87 // Force the linker to keep the symbols for various MemProf interface
88 // functions. We want to keep those in the executable in order to let the
89 // instrumented dynamic libraries access the symbol even if it is not used by
90 // the executable itself. This should help if the build system is removing dead
91 // code at link time.
92 static NOINLINE void force_interface_symbols() {
93   volatile int fake_condition = 0; // prevent dead condition elimination.
94   // clang-format off
95   switch (fake_condition) {
96     case 1: __memprof_record_access(nullptr); break;
97     case 2: __memprof_record_access_range(nullptr, 0); break;
98   }
99   // clang-format on
100 }
101 
102 static void memprof_atexit() {
103   Printf("MemProfiler exit stats:\n");
104   __memprof_print_accumulated_stats();
105 }
106 
107 static void InitializeHighMemEnd() {
108   kHighMemEnd = GetMaxUserVirtualAddress();
109   // Increase kHighMemEnd to make sure it's properly
110   // aligned together with kHighMemBeg:
111   kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
112 }
113 
114 void PrintAddressSpaceLayout() {
115   if (kHighMemBeg) {
116     Printf("|| `[%p, %p]` || HighMem    ||\n", (void *)kHighMemBeg,
117            (void *)kHighMemEnd);
118     Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg,
119            (void *)kHighShadowEnd);
120   }
121   Printf("|| `[%p, %p]` || ShadowGap  ||\n", (void *)kShadowGapBeg,
122          (void *)kShadowGapEnd);
123   if (kLowShadowBeg) {
124     Printf("|| `[%p, %p]` || LowShadow  ||\n", (void *)kLowShadowBeg,
125            (void *)kLowShadowEnd);
126     Printf("|| `[%p, %p]` || LowMem     ||\n", (void *)kLowMemBeg,
127            (void *)kLowMemEnd);
128   }
129   Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg),
130          (void *)MEM_TO_SHADOW(kLowShadowEnd));
131   if (kHighMemBeg) {
132     Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg),
133            (void *)MEM_TO_SHADOW(kHighShadowEnd));
134   }
135   Printf("\n");
136   Printf("malloc_context_size=%zu\n",
137          (uptr)common_flags()->malloc_context_size);
138 
139   Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
140   Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
141   Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
142   CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
143 }
144 
145 static bool UNUSED __local_memprof_dyninit = [] {
146   MaybeStartBackgroudThread();
147   SetSoftRssLimitExceededCallback(MemprofSoftRssLimitExceededCallback);
148 
149   return false;
150 }();
151 
152 static void MemprofInitInternal() {
153   if (LIKELY(memprof_inited))
154     return;
155   SanitizerToolName = "MemProfiler";
156   CHECK(!memprof_init_is_running && "MemProf init calls itself!");
157   memprof_init_is_running = true;
158 
159   CacheBinaryName();
160 
161   // Initialize flags. This must be done early, because most of the
162   // initialization steps look at flags().
163   InitializeFlags();
164 
165   AvoidCVE_2016_2143();
166 
167   SetMallocContextSize(common_flags()->malloc_context_size);
168 
169   InitializeHighMemEnd();
170 
171   // Make sure we are not statically linked.
172   MemprofDoesNotSupportStaticLinkage();
173 
174   // Install tool-specific callbacks in sanitizer_common.
175   AddDieCallback(MemprofDie);
176   SetCheckFailedCallback(MemprofCheckFailed);
177 
178   // Use profile name specified via the binary itself if it exists, and hasn't
179   // been overrriden by a flag at runtime.
180   if (__memprof_profile_filename[0] != 0 && !common_flags()->log_path)
181     __sanitizer_set_report_path(__memprof_profile_filename);
182   else
183     __sanitizer_set_report_path(common_flags()->log_path);
184 
185   __sanitizer::InitializePlatformEarly();
186 
187   // Re-exec ourselves if we need to set additional env or command line args.
188   MaybeReexec();
189 
190   // Setup internal allocator callback.
191   SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
192 
193   InitializeMemprofInterceptors();
194   CheckASLR();
195 
196   ReplaceSystemMalloc();
197 
198   DisableCoreDumperIfNecessary();
199 
200   InitializeShadowMemory();
201 
202   TSDInit(PlatformTSDDtor);
203 
204   InitializeAllocator();
205 
206   // On Linux MemprofThread::ThreadStart() calls malloc() that's why
207   // memprof_inited should be set to 1 prior to initializing the threads.
208   memprof_inited = 1;
209   memprof_init_is_running = false;
210 
211   if (flags()->atexit)
212     Atexit(memprof_atexit);
213 
214   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
215 
216   // interceptors
217   InitTlsSize();
218 
219   // Create main thread.
220   MemprofThread *main_thread = CreateMainThread();
221   CHECK_EQ(0, main_thread->tid());
222   force_interface_symbols(); // no-op.
223   SanitizerInitializeUnwinder();
224 
225   Symbolizer::LateInitialize();
226 
227   VReport(1, "MemProfiler Init done\n");
228 
229   memprof_init_done = 1;
230 }
231 
232 void MemprofInitTime() {
233   if (LIKELY(memprof_timestamp_inited))
234     return;
235   timespec ts;
236   timespec_get(&ts, TIME_UTC);
237   memprof_init_timestamp_s = ts.tv_sec;
238   memprof_timestamp_inited = 1;
239 }
240 
241 // Initialize as requested from some part of MemProf runtime library
242 // (interceptors, allocator, etc).
243 void MemprofInitFromRtl() { MemprofInitInternal(); }
244 
245 #if MEMPROF_DYNAMIC
246 // Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable
247 // (and thus normal initializers from .preinit_array or modules haven't run).
248 
249 class MemprofInitializer {
250 public:
251   MemprofInitializer() { MemprofInitFromRtl(); }
252 };
253 
254 static MemprofInitializer memprof_initializer;
255 #endif // MEMPROF_DYNAMIC
256 
257 } // namespace __memprof
258 
259 // ---------------------- Interface ---------------- {{{1
260 using namespace __memprof;
261 
262 // Initialize as requested from instrumented application code.
263 void __memprof_init() {
264   MemprofInitTime();
265   MemprofInitInternal();
266 }
267 
268 void __memprof_preinit() { MemprofInitInternal(); }
269 
270 void __memprof_version_mismatch_check_v1() {}
271 
272 void __memprof_record_access(void const volatile *addr) {
273   __memprof::RecordAccess((uptr)addr);
274 }
275 
276 // We only record the access on the first location in the range,
277 // since we will later accumulate the access counts across the
278 // full allocation, and we don't want to inflate the hotness from
279 // a memory intrinsic on a large range of memory.
280 // TODO: Should we do something else so we can better track utilization?
281 void __memprof_record_access_range(void const volatile *addr,
282                                    UNUSED uptr size) {
283   __memprof::RecordAccess((uptr)addr);
284 }
285 
286 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16
287 __sanitizer_unaligned_load16(const uu16 *p) {
288   __memprof_record_access(p);
289   return *p;
290 }
291 
292 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32
293 __sanitizer_unaligned_load32(const uu32 *p) {
294   __memprof_record_access(p);
295   return *p;
296 }
297 
298 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
299 __sanitizer_unaligned_load64(const uu64 *p) {
300   __memprof_record_access(p);
301   return *p;
302 }
303 
304 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
305 __sanitizer_unaligned_store16(uu16 *p, u16 x) {
306   __memprof_record_access(p);
307   *p = x;
308 }
309 
310 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
311 __sanitizer_unaligned_store32(uu32 *p, u32 x) {
312   __memprof_record_access(p);
313   *p = x;
314 }
315 
316 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
317 __sanitizer_unaligned_store64(uu64 *p, u64 x) {
318   __memprof_record_access(p);
319   *p = x;
320 }
321