1 //===-- hwasan.cpp --------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of HWAddressSanitizer. 10 // 11 // HWAddressSanitizer runtime. 12 //===----------------------------------------------------------------------===// 13 14 #include "hwasan.h" 15 #include "hwasan_checks.h" 16 #include "hwasan_dynamic_shadow.h" 17 #include "hwasan_poisoning.h" 18 #include "hwasan_report.h" 19 #include "hwasan_thread.h" 20 #include "hwasan_thread_list.h" 21 #include "sanitizer_common/sanitizer_atomic.h" 22 #include "sanitizer_common/sanitizer_common.h" 23 #include "sanitizer_common/sanitizer_flag_parser.h" 24 #include "sanitizer_common/sanitizer_flags.h" 25 #include "sanitizer_common/sanitizer_libc.h" 26 #include "sanitizer_common/sanitizer_procmaps.h" 27 #include "sanitizer_common/sanitizer_stackdepot.h" 28 #include "sanitizer_common/sanitizer_stacktrace.h" 29 #include "sanitizer_common/sanitizer_symbolizer.h" 30 #include "ubsan/ubsan_flags.h" 31 #include "ubsan/ubsan_init.h" 32 33 // ACHTUNG! No system header includes in this file. 34 35 using namespace __sanitizer; 36 37 namespace __hwasan { 38 39 static Flags hwasan_flags; 40 41 Flags *flags() { 42 return &hwasan_flags; 43 } 44 45 int hwasan_inited = 0; 46 int hwasan_instrumentation_inited = 0; 47 bool hwasan_init_is_running; 48 49 int hwasan_report_count = 0; 50 51 void Flags::SetDefaults() { 52 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; 53 #include "hwasan_flags.inc" 54 #undef HWASAN_FLAG 55 } 56 57 static void RegisterHwasanFlags(FlagParser *parser, Flags *f) { 58 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) \ 59 RegisterFlag(parser, #Name, Description, &f->Name); 60 #include "hwasan_flags.inc" 61 #undef HWASAN_FLAG 62 } 63 64 static void InitializeFlags() { 65 SetCommonFlagsDefaults(); 66 { 67 CommonFlags cf; 68 cf.CopyFrom(*common_flags()); 69 cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH"); 70 cf.malloc_context_size = 20; 71 cf.handle_ioctl = true; 72 // FIXME: test and enable. 73 cf.check_printf = false; 74 cf.intercept_tls_get_addr = true; 75 cf.exitcode = 99; 76 // 8 shadow pages ~512kB, small enough to cover common stack sizes. 77 cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8); 78 // Sigtrap is used in error reporting. 79 cf.handle_sigtrap = kHandleSignalExclusive; 80 81 #if SANITIZER_ANDROID 82 // Let platform handle other signals. It is better at reporting them then we 83 // are. 84 cf.handle_segv = kHandleSignalNo; 85 cf.handle_sigbus = kHandleSignalNo; 86 cf.handle_abort = kHandleSignalNo; 87 cf.handle_sigill = kHandleSignalNo; 88 cf.handle_sigfpe = kHandleSignalNo; 89 #endif 90 OverrideCommonFlags(cf); 91 } 92 93 Flags *f = flags(); 94 f->SetDefaults(); 95 96 FlagParser parser; 97 RegisterHwasanFlags(&parser, f); 98 RegisterCommonFlags(&parser); 99 100 #if HWASAN_CONTAINS_UBSAN 101 __ubsan::Flags *uf = __ubsan::flags(); 102 uf->SetDefaults(); 103 104 FlagParser ubsan_parser; 105 __ubsan::RegisterUbsanFlags(&ubsan_parser, uf); 106 RegisterCommonFlags(&ubsan_parser); 107 #endif 108 109 // Override from user-specified string. 110 if (__hwasan_default_options) 111 parser.ParseString(__hwasan_default_options()); 112 #if HWASAN_CONTAINS_UBSAN 113 const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions(); 114 ubsan_parser.ParseString(ubsan_default_options); 115 #endif 116 117 parser.ParseStringFromEnv("HWASAN_OPTIONS"); 118 #if HWASAN_CONTAINS_UBSAN 119 ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); 120 #endif 121 122 InitializeCommonFlags(); 123 124 if (Verbosity()) ReportUnrecognizedFlags(); 125 126 if (common_flags()->help) parser.PrintFlagDescriptions(); 127 } 128 129 static void HWAsanCheckFailed(const char *file, int line, const char *cond, 130 u64 v1, u64 v2) { 131 Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, 132 line, cond, (uptr)v1, (uptr)v2); 133 PRINT_CURRENT_STACK_CHECK(); 134 Die(); 135 } 136 137 static constexpr uptr kMemoryUsageBufferSize = 4096; 138 139 static void HwasanFormatMemoryUsage(InternalScopedString &s) { 140 HwasanThreadList &thread_list = hwasanThreadList(); 141 auto thread_stats = thread_list.GetThreadStats(); 142 auto *sds = StackDepotGetStats(); 143 AllocatorStatCounters asc; 144 GetAllocatorStats(asc); 145 s.append( 146 "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd" 147 " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd" 148 " heap: %zd", 149 internal_getpid(), GetRSS(), thread_stats.n_live_threads, 150 thread_stats.total_stack_size, 151 thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(), 152 sds->allocated, sds->n_uniq_ids, asc[AllocatorStatMapped]); 153 } 154 155 #if SANITIZER_ANDROID 156 static char *memory_usage_buffer = nullptr; 157 158 static void InitMemoryUsage() { 159 memory_usage_buffer = 160 (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string"); 161 CHECK(memory_usage_buffer); 162 memory_usage_buffer[0] = '\0'; 163 DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize, 164 memory_usage_buffer); 165 } 166 167 void UpdateMemoryUsage() { 168 if (!flags()->export_memory_stats) 169 return; 170 if (!memory_usage_buffer) 171 InitMemoryUsage(); 172 InternalScopedString s(kMemoryUsageBufferSize); 173 HwasanFormatMemoryUsage(s); 174 internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1); 175 memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0'; 176 } 177 #else 178 void UpdateMemoryUsage() {} 179 #endif 180 181 } // namespace __hwasan 182 183 using namespace __hwasan; 184 185 void __sanitizer::BufferedStackTrace::UnwindImpl( 186 uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { 187 Thread *t = GetCurrentThread(); 188 if (!t) { 189 // the thread is still being created. 190 size = 0; 191 return; 192 } 193 Unwind(max_depth, pc, bp, context, t->stack_top(), t->stack_bottom(), 194 request_fast); 195 } 196 197 struct hwasan_global { 198 s32 gv_relptr; 199 u32 info; 200 }; 201 202 static void InitGlobals(const hwasan_global *begin, const hwasan_global *end) { 203 for (auto *desc = begin; desc != end; ++desc) { 204 uptr gv = reinterpret_cast<uptr>(desc) + desc->gv_relptr; 205 uptr size = desc->info & 0xffffff; 206 uptr full_granule_size = RoundDownTo(size, 16); 207 u8 tag = desc->info >> 24; 208 TagMemoryAligned(gv, full_granule_size, tag); 209 if (size % 16) 210 TagMemoryAligned(gv + full_granule_size, 16, size % 16); 211 } 212 } 213 214 enum { NT_LLVM_HWASAN_GLOBALS = 3 }; 215 216 struct hwasan_global_note { 217 s32 begin_relptr; 218 s32 end_relptr; 219 }; 220 221 // Check that the given library meets the code model requirements for tagged 222 // globals. These properties are not checked at link time so they need to be 223 // checked at runtime. 224 static void CheckCodeModel(ElfW(Addr) base, const ElfW(Phdr) * phdr, 225 ElfW(Half) phnum) { 226 ElfW(Addr) min_addr = -1ull, max_addr = 0; 227 for (unsigned i = 0; i != phnum; ++i) { 228 if (phdr[i].p_type != PT_LOAD) 229 continue; 230 ElfW(Addr) lo = base + phdr[i].p_vaddr, hi = lo + phdr[i].p_memsz; 231 if (min_addr > lo) 232 min_addr = lo; 233 if (max_addr < hi) 234 max_addr = hi; 235 } 236 237 if (max_addr - min_addr > 1ull << 32) { 238 Report("FATAL: HWAddressSanitizer: library size exceeds 2^32\n"); 239 Die(); 240 } 241 if (max_addr > 1ull << 48) { 242 Report("FATAL: HWAddressSanitizer: library loaded above address 2^48\n"); 243 Die(); 244 } 245 } 246 247 static void InitGlobalsFromPhdrs(ElfW(Addr) base, const ElfW(Phdr) * phdr, 248 ElfW(Half) phnum) { 249 for (unsigned i = 0; i != phnum; ++i) { 250 if (phdr[i].p_type != PT_NOTE) 251 continue; 252 const char *note = reinterpret_cast<const char *>(base + phdr[i].p_vaddr); 253 const char *nend = note + phdr[i].p_memsz; 254 while (note < nend) { 255 auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(note); 256 const char *name = note + sizeof(ElfW(Nhdr)); 257 const char *desc = name + RoundUpTo(nhdr->n_namesz, 4); 258 if (nhdr->n_type != NT_LLVM_HWASAN_GLOBALS || 259 internal_strcmp(name, "LLVM") != 0) { 260 note = desc + RoundUpTo(nhdr->n_descsz, 4); 261 continue; 262 } 263 264 // Only libraries with instrumented globals need to be checked against the 265 // code model since they use relocations that aren't checked at link time. 266 CheckCodeModel(base, phdr, phnum); 267 268 auto *global_note = reinterpret_cast<const hwasan_global_note *>(desc); 269 auto *global_begin = reinterpret_cast<const hwasan_global *>( 270 note + global_note->begin_relptr); 271 auto *global_end = reinterpret_cast<const hwasan_global *>( 272 note + global_note->end_relptr); 273 InitGlobals(global_begin, global_end); 274 return; 275 } 276 } 277 } 278 279 static void InitLoadedGlobals() { 280 dl_iterate_phdr( 281 [](dl_phdr_info *info, size_t size, void *data) { 282 InitGlobalsFromPhdrs(info->dlpi_addr, info->dlpi_phdr, 283 info->dlpi_phnum); 284 return 0; 285 }, 286 nullptr); 287 } 288 289 // Prepare to run instrumented code on the main thread. 290 static void InitInstrumentation() { 291 if (hwasan_instrumentation_inited) return; 292 293 InitPrctl(); 294 295 if (!InitShadow()) { 296 Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); 297 DumpProcessMap(); 298 Die(); 299 } 300 301 InitThreads(); 302 hwasanThreadList().CreateCurrentThread(); 303 304 hwasan_instrumentation_inited = 1; 305 } 306 307 // Interface. 308 309 uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. 310 311 // This function was used by the old frame descriptor mechanism. We keep it 312 // around to avoid breaking ABI. 313 void __hwasan_init_frames(uptr beg, uptr end) {} 314 315 void __hwasan_init_static() { 316 InitShadowGOT(); 317 InitInstrumentation(); 318 319 // In the non-static code path we call dl_iterate_phdr here. But at this point 320 // libc might not have been initialized enough for dl_iterate_phdr to work. 321 // Fortunately, since this is a statically linked executable we can use the 322 // linker-defined symbol __ehdr_start to find the only relevant set of phdrs. 323 extern ElfW(Ehdr) __ehdr_start; 324 InitGlobalsFromPhdrs( 325 0, 326 reinterpret_cast<const ElfW(Phdr) *>( 327 reinterpret_cast<const char *>(&__ehdr_start) + __ehdr_start.e_phoff), 328 __ehdr_start.e_phnum); 329 } 330 331 void __hwasan_init() { 332 CHECK(!hwasan_init_is_running); 333 if (hwasan_inited) return; 334 hwasan_init_is_running = 1; 335 SanitizerToolName = "HWAddressSanitizer"; 336 337 InitTlsSize(); 338 339 CacheBinaryName(); 340 InitializeFlags(); 341 342 // Install tool-specific callbacks in sanitizer_common. 343 SetCheckFailedCallback(HWAsanCheckFailed); 344 345 __sanitizer_set_report_path(common_flags()->log_path); 346 347 AndroidTestTlsSlot(); 348 349 DisableCoreDumperIfNecessary(); 350 351 InitInstrumentation(); 352 InitLoadedGlobals(); 353 354 // Needs to be called here because flags()->random_tags might not have been 355 // initialized when InitInstrumentation() was called. 356 GetCurrentThread()->InitRandomState(); 357 358 MadviseShadow(); 359 360 SetPrintfAndReportCallback(AppendToErrorMessageBuffer); 361 // This may call libc -> needs initialized shadow. 362 AndroidLogInit(); 363 364 InitializeInterceptors(); 365 InstallDeadlySignalHandlers(HwasanOnDeadlySignal); 366 InstallAtExitHandler(); // Needs __cxa_atexit interceptor. 367 368 InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); 369 370 HwasanTSDInit(); 371 HwasanTSDThreadInit(); 372 373 HwasanAllocatorInit(); 374 375 #if HWASAN_CONTAINS_UBSAN 376 __ubsan::InitAsPlugin(); 377 #endif 378 379 VPrintf(1, "HWAddressSanitizer init done\n"); 380 381 hwasan_init_is_running = 0; 382 hwasan_inited = 1; 383 } 384 385 void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr, 386 ElfW(Half) phnum) { 387 InitGlobalsFromPhdrs(base, phdr, phnum); 388 } 389 390 void __hwasan_library_unloaded(ElfW(Addr) base, const ElfW(Phdr) * phdr, 391 ElfW(Half) phnum) { 392 for (; phnum != 0; ++phdr, --phnum) 393 if (phdr->p_type == PT_LOAD) 394 TagMemory(base + phdr->p_vaddr, phdr->p_memsz, 0); 395 } 396 397 void __hwasan_print_shadow(const void *p, uptr sz) { 398 uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); 399 uptr shadow_first = MemToShadow(ptr_raw); 400 uptr shadow_last = MemToShadow(ptr_raw + sz - 1); 401 Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw, 402 ptr_raw + sz, GetTagFromPointer((uptr)p)); 403 for (uptr s = shadow_first; s <= shadow_last; ++s) 404 Printf(" %zx: %x\n", ShadowToMem(s), *(tag_t *)s); 405 } 406 407 sptr __hwasan_test_shadow(const void *p, uptr sz) { 408 if (sz == 0) 409 return -1; 410 tag_t ptr_tag = GetTagFromPointer((uptr)p); 411 uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); 412 uptr shadow_first = MemToShadow(ptr_raw); 413 uptr shadow_last = MemToShadow(ptr_raw + sz - 1); 414 for (uptr s = shadow_first; s <= shadow_last; ++s) 415 if (*(tag_t *)s != ptr_tag) { 416 sptr offset = ShadowToMem(s) - ptr_raw; 417 return offset < 0 ? 0 : offset; 418 } 419 return -1; 420 } 421 422 u16 __sanitizer_unaligned_load16(const uu16 *p) { 423 return *p; 424 } 425 u32 __sanitizer_unaligned_load32(const uu32 *p) { 426 return *p; 427 } 428 u64 __sanitizer_unaligned_load64(const uu64 *p) { 429 return *p; 430 } 431 void __sanitizer_unaligned_store16(uu16 *p, u16 x) { 432 *p = x; 433 } 434 void __sanitizer_unaligned_store32(uu32 *p, u32 x) { 435 *p = x; 436 } 437 void __sanitizer_unaligned_store64(uu64 *p, u64 x) { 438 *p = x; 439 } 440 441 void __hwasan_loadN(uptr p, uptr sz) { 442 CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz); 443 } 444 void __hwasan_load1(uptr p) { 445 CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p); 446 } 447 void __hwasan_load2(uptr p) { 448 CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p); 449 } 450 void __hwasan_load4(uptr p) { 451 CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p); 452 } 453 void __hwasan_load8(uptr p) { 454 CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p); 455 } 456 void __hwasan_load16(uptr p) { 457 CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p); 458 } 459 460 void __hwasan_loadN_noabort(uptr p, uptr sz) { 461 CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz); 462 } 463 void __hwasan_load1_noabort(uptr p) { 464 CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p); 465 } 466 void __hwasan_load2_noabort(uptr p) { 467 CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p); 468 } 469 void __hwasan_load4_noabort(uptr p) { 470 CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p); 471 } 472 void __hwasan_load8_noabort(uptr p) { 473 CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p); 474 } 475 void __hwasan_load16_noabort(uptr p) { 476 CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p); 477 } 478 479 void __hwasan_storeN(uptr p, uptr sz) { 480 CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz); 481 } 482 void __hwasan_store1(uptr p) { 483 CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p); 484 } 485 void __hwasan_store2(uptr p) { 486 CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p); 487 } 488 void __hwasan_store4(uptr p) { 489 CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p); 490 } 491 void __hwasan_store8(uptr p) { 492 CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p); 493 } 494 void __hwasan_store16(uptr p) { 495 CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p); 496 } 497 498 void __hwasan_storeN_noabort(uptr p, uptr sz) { 499 CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz); 500 } 501 void __hwasan_store1_noabort(uptr p) { 502 CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p); 503 } 504 void __hwasan_store2_noabort(uptr p) { 505 CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p); 506 } 507 void __hwasan_store4_noabort(uptr p) { 508 CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p); 509 } 510 void __hwasan_store8_noabort(uptr p) { 511 CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p); 512 } 513 void __hwasan_store16_noabort(uptr p) { 514 CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p); 515 } 516 517 void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { 518 TagMemoryAligned(p, sz, tag); 519 } 520 521 uptr __hwasan_tag_pointer(uptr p, u8 tag) { 522 return AddTagToPointer(p, tag); 523 } 524 525 void __hwasan_handle_longjmp(const void *sp_dst) { 526 uptr dst = (uptr)sp_dst; 527 // HWASan does not support tagged SP. 528 CHECK(GetTagFromPointer(dst) == 0); 529 530 uptr sp = (uptr)__builtin_frame_address(0); 531 static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M 532 if (dst < sp || dst - sp > kMaxExpectedCleanupSize) { 533 Report( 534 "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: " 535 "stack top: %p; target %p; distance: %p (%zd)\n" 536 "False positive error reports may follow\n", 537 (void *)sp, (void *)dst, dst - sp); 538 return; 539 } 540 TagMemory(sp, dst - sp, 0); 541 } 542 543 void __hwasan_handle_vfork(const void *sp_dst) { 544 uptr sp = (uptr)sp_dst; 545 Thread *t = GetCurrentThread(); 546 CHECK(t); 547 uptr top = t->stack_top(); 548 uptr bottom = t->stack_bottom(); 549 if (top == 0 || bottom == 0 || sp < bottom || sp >= top) { 550 Report( 551 "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: " 552 "stack top: %zx; current %zx; bottom: %zx \n" 553 "False positive error reports may follow\n", 554 top, sp, bottom); 555 return; 556 } 557 TagMemory(bottom, sp - bottom, 0); 558 } 559 560 extern "C" void *__hwasan_extra_spill_area() { 561 Thread *t = GetCurrentThread(); 562 return &t->vfork_spill(); 563 } 564 565 void __hwasan_print_memory_usage() { 566 InternalScopedString s(kMemoryUsageBufferSize); 567 HwasanFormatMemoryUsage(s); 568 Printf("%s\n", s.data()); 569 } 570 571 static const u8 kFallbackTag = 0xBB; 572 573 u8 __hwasan_generate_tag() { 574 Thread *t = GetCurrentThread(); 575 if (!t) return kFallbackTag; 576 return t->GenerateRandomTag(); 577 } 578 579 #if !SANITIZER_SUPPORTS_WEAK_HOOKS 580 extern "C" { 581 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE 582 const char* __hwasan_default_options() { return ""; } 583 } // extern "C" 584 #endif 585 586 extern "C" { 587 SANITIZER_INTERFACE_ATTRIBUTE 588 void __sanitizer_print_stack_trace() { 589 GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); 590 stack.Print(); 591 } 592 } // extern "C" 593