1 //===-- hwasan_linux.cc -----------------------------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and
12 /// FreeBSD-specific code.
13 ///
14 //===----------------------------------------------------------------------===//
15
16 #include "sanitizer_common/sanitizer_platform.h"
17 #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
18
19 #include "hwasan.h"
20 #include "hwasan_dynamic_shadow.h"
21 #include "hwasan_interface_internal.h"
22 #include "hwasan_mapping.h"
23 #include "hwasan_report.h"
24 #include "hwasan_thread.h"
25 #include "hwasan_thread_list.h"
26
27 #include <dlfcn.h>
28 #include <elf.h>
29 #include <link.h>
30 #include <pthread.h>
31 #include <signal.h>
32 #include <stdio.h>
33 #include <stdlib.h>
34 #include <sys/resource.h>
35 #include <sys/time.h>
36 #include <unistd.h>
37 #include <unwind.h>
38
39 #include "sanitizer_common/sanitizer_common.h"
40 #include "sanitizer_common/sanitizer_procmaps.h"
41
42 #if HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID
43 SANITIZER_INTERFACE_ATTRIBUTE
44 THREADLOCAL uptr __hwasan_tls;
45 #endif
46
47 namespace __hwasan {
48
ReserveShadowMemoryRange(uptr beg,uptr end,const char * name)49 static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
50 CHECK_EQ((beg % GetMmapGranularity()), 0);
51 CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
52 uptr size = end - beg + 1;
53 DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
54 if (!MmapFixedNoReserve(beg, size, name)) {
55 Report(
56 "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
57 "Perhaps you're using ulimit -v\n",
58 size);
59 Abort();
60 }
61 }
62
ProtectGap(uptr addr,uptr size)63 static void ProtectGap(uptr addr, uptr size) {
64 if (!size)
65 return;
66 void *res = MmapFixedNoAccess(addr, size, "shadow gap");
67 if (addr == (uptr)res)
68 return;
69 // A few pages at the start of the address space can not be protected.
70 // But we really want to protect as much as possible, to prevent this memory
71 // being returned as a result of a non-FIXED mmap().
72 if (addr == 0) {
73 uptr step = GetMmapGranularity();
74 while (size > step) {
75 addr += step;
76 size -= step;
77 void *res = MmapFixedNoAccess(addr, size, "shadow gap");
78 if (addr == (uptr)res)
79 return;
80 }
81 }
82
83 Report(
84 "ERROR: Failed to protect shadow gap [%p, %p]. "
85 "HWASan cannot proceed correctly. ABORTING.\n", (void *)addr,
86 (void *)(addr + size));
87 DumpProcessMap();
88 Die();
89 }
90
91 static uptr kLowMemStart;
92 static uptr kLowMemEnd;
93 static uptr kLowShadowEnd;
94 static uptr kLowShadowStart;
95 static uptr kHighShadowStart;
96 static uptr kHighShadowEnd;
97 static uptr kHighMemStart;
98 static uptr kHighMemEnd;
99
PrintRange(uptr start,uptr end,const char * name)100 static void PrintRange(uptr start, uptr end, const char *name) {
101 Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
102 }
103
PrintAddressSpaceLayout()104 static void PrintAddressSpaceLayout() {
105 PrintRange(kHighMemStart, kHighMemEnd, "HighMem");
106 if (kHighShadowEnd + 1 < kHighMemStart)
107 PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap");
108 else
109 CHECK_EQ(kHighShadowEnd + 1, kHighMemStart);
110 PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow");
111 if (kLowShadowEnd + 1 < kHighShadowStart)
112 PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap");
113 else
114 CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
115 PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
116 if (kLowMemEnd + 1 < kLowShadowStart)
117 PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap");
118 else
119 CHECK_EQ(kLowMemEnd + 1, kLowShadowStart);
120 PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
121 CHECK_EQ(0, kLowMemStart);
122 }
123
GetHighMemEnd()124 static uptr GetHighMemEnd() {
125 // HighMem covers the upper part of the address space.
126 uptr max_address = GetMaxUserVirtualAddress();
127 // Adjust max address to make sure that kHighMemEnd and kHighMemStart are
128 // properly aligned:
129 max_address |= (GetMmapGranularity() << kShadowScale) - 1;
130 return max_address;
131 }
132
InitializeShadowBaseAddress(uptr shadow_size_bytes)133 static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
134 __hwasan_shadow_memory_dynamic_address =
135 FindDynamicShadowStart(shadow_size_bytes);
136 }
137
InitShadow()138 bool InitShadow() {
139 // Define the entire memory range.
140 kHighMemEnd = GetHighMemEnd();
141
142 // Determine shadow memory base offset.
143 InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd));
144
145 // Place the low memory first.
146 kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1;
147 kLowMemStart = 0;
148
149 // Define the low shadow based on the already placed low memory.
150 kLowShadowEnd = MemToShadow(kLowMemEnd);
151 kLowShadowStart = __hwasan_shadow_memory_dynamic_address;
152
153 // High shadow takes whatever memory is left up there (making sure it is not
154 // interfering with low memory in the fixed case).
155 kHighShadowEnd = MemToShadow(kHighMemEnd);
156 kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1;
157
158 // High memory starts where allocated shadow allows.
159 kHighMemStart = ShadowToMem(kHighShadowStart);
160
161 // Check the sanity of the defined memory ranges (there might be gaps).
162 CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
163 CHECK_GT(kHighMemStart, kHighShadowEnd);
164 CHECK_GT(kHighShadowEnd, kHighShadowStart);
165 CHECK_GT(kHighShadowStart, kLowMemEnd);
166 CHECK_GT(kLowMemEnd, kLowMemStart);
167 CHECK_GT(kLowShadowEnd, kLowShadowStart);
168 CHECK_GT(kLowShadowStart, kLowMemEnd);
169
170 if (Verbosity())
171 PrintAddressSpaceLayout();
172
173 // Reserve shadow memory.
174 ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow");
175 ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow");
176
177 // Protect all the gaps.
178 ProtectGap(0, Min(kLowMemStart, kLowShadowStart));
179 if (kLowMemEnd + 1 < kLowShadowStart)
180 ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1);
181 if (kLowShadowEnd + 1 < kHighShadowStart)
182 ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1);
183 if (kHighShadowEnd + 1 < kHighMemStart)
184 ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1);
185
186 return true;
187 }
188
InitThreads()189 void InitThreads() {
190 CHECK(__hwasan_shadow_memory_dynamic_address);
191 uptr guard_page_size = GetMmapGranularity();
192 uptr thread_space_start =
193 __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment);
194 uptr thread_space_end =
195 __hwasan_shadow_memory_dynamic_address - guard_page_size;
196 ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1,
197 "hwasan threads");
198 ProtectGap(thread_space_end,
199 __hwasan_shadow_memory_dynamic_address - thread_space_end);
200 InitThreadList(thread_space_start, thread_space_end - thread_space_start);
201 }
202
MadviseShadowRegion(uptr beg,uptr end)203 static void MadviseShadowRegion(uptr beg, uptr end) {
204 uptr size = end - beg + 1;
205 if (common_flags()->no_huge_pages_for_shadow)
206 NoHugePagesInRegion(beg, size);
207 if (common_flags()->use_madv_dontdump)
208 DontDumpShadowMemory(beg, size);
209 }
210
MadviseShadow()211 void MadviseShadow() {
212 MadviseShadowRegion(kLowShadowStart, kLowShadowEnd);
213 MadviseShadowRegion(kHighShadowStart, kHighShadowEnd);
214 }
215
MemIsApp(uptr p)216 bool MemIsApp(uptr p) {
217 CHECK(GetTagFromPointer(p) == 0);
218 return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
219 }
220
HwasanAtExit(void)221 static void HwasanAtExit(void) {
222 if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
223 ReportStats();
224 if (hwasan_report_count > 0) {
225 // ReportAtExitStatistics();
226 if (common_flags()->exitcode)
227 internal__exit(common_flags()->exitcode);
228 }
229 }
230
InstallAtExitHandler()231 void InstallAtExitHandler() {
232 atexit(HwasanAtExit);
233 }
234
235 // ---------------------- TSD ---------------- {{{1
236
__hwasan_thread_enter()237 extern "C" void __hwasan_thread_enter() {
238 hwasanThreadList().CreateCurrentThread();
239 }
240
__hwasan_thread_exit()241 extern "C" void __hwasan_thread_exit() {
242 Thread *t = GetCurrentThread();
243 // Make sure that signal handler can not see a stale current thread pointer.
244 atomic_signal_fence(memory_order_seq_cst);
245 if (t)
246 hwasanThreadList().ReleaseThread(t);
247 }
248
249 #if HWASAN_WITH_INTERCEPTORS
250 static pthread_key_t tsd_key;
251 static bool tsd_key_inited = false;
252
HwasanTSDThreadInit()253 void HwasanTSDThreadInit() {
254 if (tsd_key_inited)
255 CHECK_EQ(0, pthread_setspecific(tsd_key,
256 (void *)GetPthreadDestructorIterations()));
257 }
258
HwasanTSDDtor(void * tsd)259 void HwasanTSDDtor(void *tsd) {
260 uptr iterations = (uptr)tsd;
261 if (iterations > 1) {
262 CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1)));
263 return;
264 }
265 __hwasan_thread_exit();
266 }
267
HwasanTSDInit()268 void HwasanTSDInit() {
269 CHECK(!tsd_key_inited);
270 tsd_key_inited = true;
271 CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor));
272 }
273 #else
HwasanTSDInit()274 void HwasanTSDInit() {}
HwasanTSDThreadInit()275 void HwasanTSDThreadInit() {}
276 #endif
277
278 #if SANITIZER_ANDROID
GetCurrentThreadLongPtr()279 uptr *GetCurrentThreadLongPtr() {
280 return (uptr *)get_android_tls_ptr();
281 }
282 #else
GetCurrentThreadLongPtr()283 uptr *GetCurrentThreadLongPtr() {
284 return &__hwasan_tls;
285 }
286 #endif
287
288 #if SANITIZER_ANDROID
AndroidTestTlsSlot()289 void AndroidTestTlsSlot() {
290 uptr kMagicValue = 0x010203040A0B0C0D;
291 *(uptr *)get_android_tls_ptr() = kMagicValue;
292 dlerror();
293 if (*(uptr *)get_android_tls_ptr() != kMagicValue) {
294 Printf(
295 "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used "
296 "for dlerror().\n");
297 Die();
298 }
299 }
300 #else
AndroidTestTlsSlot()301 void AndroidTestTlsSlot() {}
302 #endif
303
GetCurrentThread()304 Thread *GetCurrentThread() {
305 uptr *ThreadLong = GetCurrentThreadLongPtr();
306 #if HWASAN_WITH_INTERCEPTORS
307 if (!*ThreadLong)
308 __hwasan_thread_enter();
309 #endif
310 auto *R = (StackAllocationsRingBuffer *)ThreadLong;
311 return hwasanThreadList().GetThreadByBufferAddress((uptr)(R->Next()));
312 }
313
314 struct AccessInfo {
315 uptr addr;
316 uptr size;
317 bool is_store;
318 bool is_load;
319 bool recover;
320 };
321
GetAccessInfo(siginfo_t * info,ucontext_t * uc)322 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
323 // Access type is passed in a platform dependent way (see below) and encoded
324 // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
325 // recoverable. Valid values of Y are 0 to 4, which are interpreted as
326 // log2(access_size), and 0xF, which means that access size is passed via
327 // platform dependent register (see below).
328 #if defined(__aarch64__)
329 // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
330 // access size is stored in X1 register. Access address is always in X0
331 // register.
332 uptr pc = (uptr)info->si_addr;
333 const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
334 if ((code & 0xff00) != 0x900)
335 return AccessInfo{}; // Not ours.
336
337 const bool is_store = code & 0x10;
338 const bool recover = code & 0x20;
339 const uptr addr = uc->uc_mcontext.regs[0];
340 const unsigned size_log = code & 0xf;
341 if (size_log > 4 && size_log != 0xf)
342 return AccessInfo{}; // Not ours.
343 const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
344
345 #elif defined(__x86_64__)
346 // Access type is encoded in the instruction following INT3 as
347 // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
348 // RSI register. Access address is always in RDI register.
349 uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
350 uint8_t *nop = (uint8_t*)pc;
351 if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 ||
352 *(nop + 3) < 0x40)
353 return AccessInfo{}; // Not ours.
354 const unsigned code = *(nop + 3);
355
356 const bool is_store = code & 0x10;
357 const bool recover = code & 0x20;
358 const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
359 const unsigned size_log = code & 0xf;
360 if (size_log > 4 && size_log != 0xf)
361 return AccessInfo{}; // Not ours.
362 const uptr size =
363 size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
364
365 #else
366 # error Unsupported architecture
367 #endif
368
369 return AccessInfo{addr, size, is_store, !is_store, recover};
370 }
371
HwasanOnSIGTRAP(int signo,siginfo_t * info,ucontext_t * uc)372 static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
373 AccessInfo ai = GetAccessInfo(info, uc);
374 if (!ai.is_store && !ai.is_load)
375 return false;
376
377 InternalMmapVector<BufferedStackTrace> stack_buffer(1);
378 BufferedStackTrace *stack = stack_buffer.data();
379 stack->Reset();
380 SignalContext sig{info, uc};
381 GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc),
382 sig.bp, uc, common_flags()->fast_unwind_on_fatal);
383
384 ++hwasan_report_count;
385
386 bool fatal = flags()->halt_on_error || !ai.recover;
387 ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal);
388
389 #if defined(__aarch64__)
390 uc->uc_mcontext.pc += 4;
391 #elif defined(__x86_64__)
392 #else
393 # error Unsupported architecture
394 #endif
395 return true;
396 }
397
OnStackUnwind(const SignalContext & sig,const void *,BufferedStackTrace * stack)398 static void OnStackUnwind(const SignalContext &sig, const void *,
399 BufferedStackTrace *stack) {
400 GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc),
401 sig.bp, sig.context, common_flags()->fast_unwind_on_fatal);
402 }
403
HwasanOnDeadlySignal(int signo,void * info,void * context)404 void HwasanOnDeadlySignal(int signo, void *info, void *context) {
405 // Probably a tag mismatch.
406 if (signo == SIGTRAP)
407 if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t*)context))
408 return;
409
410 HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
411 }
412
413
414 } // namespace __hwasan
415
416 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
417