1 //===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and 11 /// FreeBSD-specific code. 12 /// 13 //===----------------------------------------------------------------------===// 14 15 #include "sanitizer_common/sanitizer_platform.h" 16 #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD 17 18 #include "hwasan.h" 19 #include "hwasan_dynamic_shadow.h" 20 #include "hwasan_interface_internal.h" 21 #include "hwasan_mapping.h" 22 #include "hwasan_report.h" 23 #include "hwasan_thread.h" 24 #include "hwasan_thread_list.h" 25 26 #include <dlfcn.h> 27 #include <elf.h> 28 #include <link.h> 29 #include <pthread.h> 30 #include <signal.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <sys/resource.h> 34 #include <sys/time.h> 35 #include <unistd.h> 36 #include <unwind.h> 37 #include <sys/prctl.h> 38 #include <errno.h> 39 40 #include "sanitizer_common/sanitizer_common.h" 41 #include "sanitizer_common/sanitizer_procmaps.h" 42 43 // Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID. 44 // 45 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF 46 // Not currently tested. 47 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON 48 // Integration tests downstream exist. 49 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF 50 // Tested with check-hwasan on x86_64-linux. 51 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON 52 // Tested with check-hwasan on aarch64-linux-android. 53 #if !SANITIZER_ANDROID 54 SANITIZER_INTERFACE_ATTRIBUTE 55 THREADLOCAL uptr __hwasan_tls; 56 #endif 57 58 namespace __hwasan { 59 60 // With the zero shadow base we can not actually map pages starting from 0. 61 // This constant is somewhat arbitrary. 62 constexpr uptr kZeroBaseShadowStart = 0; 63 constexpr uptr kZeroBaseMaxShadowStart = 1 << 18; 64 65 static void ProtectGap(uptr addr, uptr size) { 66 __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart, 67 kZeroBaseMaxShadowStart); 68 } 69 70 uptr kLowMemStart; 71 uptr kLowMemEnd; 72 uptr kLowShadowEnd; 73 uptr kLowShadowStart; 74 uptr kHighShadowStart; 75 uptr kHighShadowEnd; 76 uptr kHighMemStart; 77 uptr kHighMemEnd; 78 79 static void PrintRange(uptr start, uptr end, const char *name) { 80 Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name); 81 } 82 83 static void PrintAddressSpaceLayout() { 84 PrintRange(kHighMemStart, kHighMemEnd, "HighMem"); 85 if (kHighShadowEnd + 1 < kHighMemStart) 86 PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap"); 87 else 88 CHECK_EQ(kHighShadowEnd + 1, kHighMemStart); 89 PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow"); 90 if (kLowShadowEnd + 1 < kHighShadowStart) 91 PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap"); 92 else 93 CHECK_EQ(kLowMemEnd + 1, kHighShadowStart); 94 PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow"); 95 if (kLowMemEnd + 1 < kLowShadowStart) 96 PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap"); 97 else 98 CHECK_EQ(kLowMemEnd + 1, kLowShadowStart); 99 PrintRange(kLowMemStart, kLowMemEnd, "LowMem"); 100 CHECK_EQ(0, kLowMemStart); 101 } 102 103 static uptr GetHighMemEnd() { 104 // HighMem covers the upper part of the address space. 105 uptr max_address = GetMaxUserVirtualAddress(); 106 // Adjust max address to make sure that kHighMemEnd and kHighMemStart are 107 // properly aligned: 108 max_address |= (GetMmapGranularity() << kShadowScale) - 1; 109 return max_address; 110 } 111 112 static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { 113 __hwasan_shadow_memory_dynamic_address = 114 FindDynamicShadowStart(shadow_size_bytes); 115 } 116 117 void InitPrctl() { 118 #define PR_SET_TAGGED_ADDR_CTRL 55 119 #define PR_GET_TAGGED_ADDR_CTRL 56 120 #define PR_TAGGED_ADDR_ENABLE (1UL << 0) 121 // Check we're running on a kernel that can use the tagged address ABI. 122 int local_errno = 0; 123 if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), 124 &local_errno) && 125 local_errno == EINVAL) { 126 #if SANITIZER_ANDROID 127 // Some older Android kernels have the tagged pointer ABI on 128 // unconditionally, and hence don't have the tagged-addr prctl while still 129 // allow the ABI. 130 // If targeting Android and the prctl is not around we assume this is the 131 // case. 132 return; 133 #else 134 if (flags()->fail_without_syscall_abi) { 135 Printf( 136 "FATAL: " 137 "HWAddressSanitizer requires a kernel with tagged address ABI.\n"); 138 Die(); 139 } 140 #endif 141 } 142 143 // Turn on the tagged address ABI. 144 if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, 145 PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) || 146 !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) && 147 flags()->fail_without_syscall_abi) { 148 Printf( 149 "FATAL: HWAddressSanitizer failed to enable tagged address syscall " 150 "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " 151 "configuration.\n"); 152 Die(); 153 } 154 #undef PR_SET_TAGGED_ADDR_CTRL 155 #undef PR_GET_TAGGED_ADDR_CTRL 156 #undef PR_TAGGED_ADDR_ENABLE 157 } 158 159 bool InitShadow() { 160 // Define the entire memory range. 161 kHighMemEnd = GetHighMemEnd(); 162 163 // Determine shadow memory base offset. 164 InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd)); 165 166 // Place the low memory first. 167 kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1; 168 kLowMemStart = 0; 169 170 // Define the low shadow based on the already placed low memory. 171 kLowShadowEnd = MemToShadow(kLowMemEnd); 172 kLowShadowStart = __hwasan_shadow_memory_dynamic_address; 173 174 // High shadow takes whatever memory is left up there (making sure it is not 175 // interfering with low memory in the fixed case). 176 kHighShadowEnd = MemToShadow(kHighMemEnd); 177 kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1; 178 179 // High memory starts where allocated shadow allows. 180 kHighMemStart = ShadowToMem(kHighShadowStart); 181 182 // Check the sanity of the defined memory ranges (there might be gaps). 183 CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0); 184 CHECK_GT(kHighMemStart, kHighShadowEnd); 185 CHECK_GT(kHighShadowEnd, kHighShadowStart); 186 CHECK_GT(kHighShadowStart, kLowMemEnd); 187 CHECK_GT(kLowMemEnd, kLowMemStart); 188 CHECK_GT(kLowShadowEnd, kLowShadowStart); 189 CHECK_GT(kLowShadowStart, kLowMemEnd); 190 191 if (Verbosity()) 192 PrintAddressSpaceLayout(); 193 194 // Reserve shadow memory. 195 ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow"); 196 ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow"); 197 198 // Protect all the gaps. 199 ProtectGap(0, Min(kLowMemStart, kLowShadowStart)); 200 if (kLowMemEnd + 1 < kLowShadowStart) 201 ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1); 202 if (kLowShadowEnd + 1 < kHighShadowStart) 203 ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1); 204 if (kHighShadowEnd + 1 < kHighMemStart) 205 ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1); 206 207 return true; 208 } 209 210 void InitThreads() { 211 CHECK(__hwasan_shadow_memory_dynamic_address); 212 uptr guard_page_size = GetMmapGranularity(); 213 uptr thread_space_start = 214 __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment); 215 uptr thread_space_end = 216 __hwasan_shadow_memory_dynamic_address - guard_page_size; 217 ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1, 218 "hwasan threads", /*madvise_shadow*/ false); 219 ProtectGap(thread_space_end, 220 __hwasan_shadow_memory_dynamic_address - thread_space_end); 221 InitThreadList(thread_space_start, thread_space_end - thread_space_start); 222 } 223 224 bool MemIsApp(uptr p) { 225 CHECK(GetTagFromPointer(p) == 0); 226 return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd); 227 } 228 229 static void HwasanAtExit(void) { 230 if (common_flags()->print_module_map) 231 DumpProcessMap(); 232 if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) 233 ReportStats(); 234 if (hwasan_report_count > 0) { 235 // ReportAtExitStatistics(); 236 if (common_flags()->exitcode) 237 internal__exit(common_flags()->exitcode); 238 } 239 } 240 241 void InstallAtExitHandler() { 242 atexit(HwasanAtExit); 243 } 244 245 // ---------------------- TSD ---------------- {{{1 246 247 extern "C" void __hwasan_thread_enter() { 248 hwasanThreadList().CreateCurrentThread()->InitRandomState(); 249 } 250 251 extern "C" void __hwasan_thread_exit() { 252 Thread *t = GetCurrentThread(); 253 // Make sure that signal handler can not see a stale current thread pointer. 254 atomic_signal_fence(memory_order_seq_cst); 255 if (t) 256 hwasanThreadList().ReleaseThread(t); 257 } 258 259 #if HWASAN_WITH_INTERCEPTORS 260 static pthread_key_t tsd_key; 261 static bool tsd_key_inited = false; 262 263 void HwasanTSDThreadInit() { 264 if (tsd_key_inited) 265 CHECK_EQ(0, pthread_setspecific(tsd_key, 266 (void *)GetPthreadDestructorIterations())); 267 } 268 269 void HwasanTSDDtor(void *tsd) { 270 uptr iterations = (uptr)tsd; 271 if (iterations > 1) { 272 CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1))); 273 return; 274 } 275 __hwasan_thread_exit(); 276 } 277 278 void HwasanTSDInit() { 279 CHECK(!tsd_key_inited); 280 tsd_key_inited = true; 281 CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor)); 282 } 283 #else 284 void HwasanTSDInit() {} 285 void HwasanTSDThreadInit() {} 286 #endif 287 288 #if SANITIZER_ANDROID 289 uptr *GetCurrentThreadLongPtr() { 290 return (uptr *)get_android_tls_ptr(); 291 } 292 #else 293 uptr *GetCurrentThreadLongPtr() { 294 return &__hwasan_tls; 295 } 296 #endif 297 298 #if SANITIZER_ANDROID 299 void AndroidTestTlsSlot() { 300 uptr kMagicValue = 0x010203040A0B0C0D; 301 uptr *tls_ptr = GetCurrentThreadLongPtr(); 302 uptr old_value = *tls_ptr; 303 *tls_ptr = kMagicValue; 304 dlerror(); 305 if (*(uptr *)get_android_tls_ptr() != kMagicValue) { 306 Printf( 307 "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used " 308 "for dlerror().\n"); 309 Die(); 310 } 311 *tls_ptr = old_value; 312 } 313 #else 314 void AndroidTestTlsSlot() {} 315 #endif 316 317 Thread *GetCurrentThread() { 318 uptr *ThreadLongPtr = GetCurrentThreadLongPtr(); 319 if (UNLIKELY(*ThreadLongPtr == 0)) 320 return nullptr; 321 auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr; 322 return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next()); 323 } 324 325 struct AccessInfo { 326 uptr addr; 327 uptr size; 328 bool is_store; 329 bool is_load; 330 bool recover; 331 }; 332 333 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { 334 // Access type is passed in a platform dependent way (see below) and encoded 335 // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is 336 // recoverable. Valid values of Y are 0 to 4, which are interpreted as 337 // log2(access_size), and 0xF, which means that access size is passed via 338 // platform dependent register (see below). 339 #if defined(__aarch64__) 340 // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF, 341 // access size is stored in X1 register. Access address is always in X0 342 // register. 343 uptr pc = (uptr)info->si_addr; 344 const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff; 345 if ((code & 0xff00) != 0x900) 346 return AccessInfo{}; // Not ours. 347 348 const bool is_store = code & 0x10; 349 const bool recover = code & 0x20; 350 const uptr addr = uc->uc_mcontext.regs[0]; 351 const unsigned size_log = code & 0xf; 352 if (size_log > 4 && size_log != 0xf) 353 return AccessInfo{}; // Not ours. 354 const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log; 355 356 #elif defined(__x86_64__) 357 // Access type is encoded in the instruction following INT3 as 358 // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in 359 // RSI register. Access address is always in RDI register. 360 uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP]; 361 uint8_t *nop = (uint8_t*)pc; 362 if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 || 363 *(nop + 3) < 0x40) 364 return AccessInfo{}; // Not ours. 365 const unsigned code = *(nop + 3); 366 367 const bool is_store = code & 0x10; 368 const bool recover = code & 0x20; 369 const uptr addr = uc->uc_mcontext.gregs[REG_RDI]; 370 const unsigned size_log = code & 0xf; 371 if (size_log > 4 && size_log != 0xf) 372 return AccessInfo{}; // Not ours. 373 const uptr size = 374 size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log; 375 376 #else 377 # error Unsupported architecture 378 #endif 379 380 return AccessInfo{addr, size, is_store, !is_store, recover}; 381 } 382 383 static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, 384 ucontext_t *uc, uptr *registers_frame = nullptr) { 385 InternalMmapVector<BufferedStackTrace> stack_buffer(1); 386 BufferedStackTrace *stack = stack_buffer.data(); 387 stack->Reset(); 388 stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); 389 390 // The second stack frame contains the failure __hwasan_check function, as 391 // we have a stack frame for the registers saved in __hwasan_tag_mismatch that 392 // we wish to ignore. This (currently) only occurs on AArch64, as x64 393 // implementations use SIGTRAP to implement the failure, and thus do not go 394 // through the stack saver. 395 if (registers_frame && stack->trace && stack->size > 0) { 396 stack->trace++; 397 stack->size--; 398 } 399 400 bool fatal = flags()->halt_on_error || !ai.recover; 401 ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, 402 registers_frame); 403 } 404 405 static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { 406 AccessInfo ai = GetAccessInfo(info, uc); 407 if (!ai.is_store && !ai.is_load) 408 return false; 409 410 SignalContext sig{info, uc}; 411 HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc); 412 413 #if defined(__aarch64__) 414 uc->uc_mcontext.pc += 4; 415 #elif defined(__x86_64__) 416 #else 417 # error Unsupported architecture 418 #endif 419 return true; 420 } 421 422 static void OnStackUnwind(const SignalContext &sig, const void *, 423 BufferedStackTrace *stack) { 424 stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context, 425 common_flags()->fast_unwind_on_fatal); 426 } 427 428 void HwasanOnDeadlySignal(int signo, void *info, void *context) { 429 // Probably a tag mismatch. 430 if (signo == SIGTRAP) 431 if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t*)context)) 432 return; 433 434 HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr); 435 } 436 437 438 } // namespace __hwasan 439 440 // Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the 441 // rest of the mismatch handling code (C++). 442 void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame, 443 size_t outsize) { 444 __hwasan::AccessInfo ai; 445 ai.is_store = access_info & 0x10; 446 ai.is_load = !ai.is_store; 447 ai.recover = access_info & 0x20; 448 ai.addr = addr; 449 if ((access_info & 0xf) == 0xf) 450 ai.size = outsize; 451 else 452 ai.size = 1 << (access_info & 0xf); 453 454 __hwasan::HandleTagMismatch(ai, (uptr)__builtin_return_address(0), 455 (uptr)__builtin_frame_address(0), nullptr, 456 registers_frame); 457 __builtin_unreachable(); 458 } 459 460 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD 461