1 //===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and 11 /// FreeBSD-specific code. 12 /// 13 //===----------------------------------------------------------------------===// 14 15 #include "sanitizer_common/sanitizer_platform.h" 16 #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD 17 18 #include "hwasan.h" 19 #include "hwasan_dynamic_shadow.h" 20 #include "hwasan_interface_internal.h" 21 #include "hwasan_mapping.h" 22 #include "hwasan_report.h" 23 #include "hwasan_thread.h" 24 #include "hwasan_thread_list.h" 25 26 #include <dlfcn.h> 27 #include <elf.h> 28 #include <link.h> 29 #include <pthread.h> 30 #include <signal.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <sys/resource.h> 34 #include <sys/time.h> 35 #include <unistd.h> 36 #include <unwind.h> 37 #include <sys/prctl.h> 38 #include <errno.h> 39 40 #include "sanitizer_common/sanitizer_common.h" 41 #include "sanitizer_common/sanitizer_procmaps.h" 42 43 // Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID. 44 // 45 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF 46 // Not currently tested. 47 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON 48 // Integration tests downstream exist. 49 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF 50 // Tested with check-hwasan on x86_64-linux. 51 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON 52 // Tested with check-hwasan on aarch64-linux-android. 53 #if !SANITIZER_ANDROID 54 SANITIZER_INTERFACE_ATTRIBUTE 55 THREADLOCAL uptr __hwasan_tls; 56 #endif 57 58 namespace __hwasan { 59 60 // With the zero shadow base we can not actually map pages starting from 0. 61 // This constant is somewhat arbitrary. 62 constexpr uptr kZeroBaseShadowStart = 0; 63 constexpr uptr kZeroBaseMaxShadowStart = 1 << 18; 64 65 static void ProtectGap(uptr addr, uptr size) { 66 __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart, 67 kZeroBaseMaxShadowStart); 68 } 69 70 uptr kLowMemStart; 71 uptr kLowMemEnd; 72 uptr kLowShadowEnd; 73 uptr kLowShadowStart; 74 uptr kHighShadowStart; 75 uptr kHighShadowEnd; 76 uptr kHighMemStart; 77 uptr kHighMemEnd; 78 79 uptr kAliasRegionStart; // Always 0 when aliases aren't used. 80 81 static void PrintRange(uptr start, uptr end, const char *name) { 82 Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name); 83 } 84 85 static void PrintAddressSpaceLayout() { 86 PrintRange(kHighMemStart, kHighMemEnd, "HighMem"); 87 if (kHighShadowEnd + 1 < kHighMemStart) 88 PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap"); 89 else 90 CHECK_EQ(kHighShadowEnd + 1, kHighMemStart); 91 PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow"); 92 if (kLowShadowEnd + 1 < kHighShadowStart) 93 PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap"); 94 else 95 CHECK_EQ(kLowMemEnd + 1, kHighShadowStart); 96 PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow"); 97 if (kLowMemEnd + 1 < kLowShadowStart) 98 PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap"); 99 else 100 CHECK_EQ(kLowMemEnd + 1, kLowShadowStart); 101 PrintRange(kLowMemStart, kLowMemEnd, "LowMem"); 102 CHECK_EQ(0, kLowMemStart); 103 } 104 105 static uptr GetHighMemEnd() { 106 // HighMem covers the upper part of the address space. 107 uptr max_address = GetMaxUserVirtualAddress(); 108 // Adjust max address to make sure that kHighMemEnd and kHighMemStart are 109 // properly aligned: 110 max_address |= (GetMmapGranularity() << kShadowScale) - 1; 111 return max_address; 112 } 113 114 static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { 115 __hwasan_shadow_memory_dynamic_address = 116 FindDynamicShadowStart(shadow_size_bytes); 117 } 118 119 void InitPrctl() { 120 #define PR_SET_TAGGED_ADDR_CTRL 55 121 #define PR_GET_TAGGED_ADDR_CTRL 56 122 #define PR_TAGGED_ADDR_ENABLE (1UL << 0) 123 // Check we're running on a kernel that can use the tagged address ABI. 124 int local_errno = 0; 125 if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), 126 &local_errno) && 127 local_errno == EINVAL) { 128 # if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE) 129 // Some older Android kernels have the tagged pointer ABI on 130 // unconditionally, and hence don't have the tagged-addr prctl while still 131 // allow the ABI. 132 // If targeting Android and the prctl is not around we assume this is the 133 // case. 134 return; 135 # else 136 if (flags()->fail_without_syscall_abi) { 137 Printf( 138 "FATAL: " 139 "HWAddressSanitizer requires a kernel with tagged address ABI.\n"); 140 Die(); 141 } 142 # endif 143 } 144 145 // Turn on the tagged address ABI. 146 if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, 147 PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) || 148 !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) { 149 # if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) 150 // Try the new prctl API for Intel LAM. The API is based on a currently 151 // unsubmitted patch to the Linux kernel (as of May 2021) and is thus 152 // subject to change. Patch is here: 153 // https://lore.kernel.org/linux-mm/[email protected]/ 154 int tag_bits = kTagBits; 155 int tag_shift = kAddressTagShift; 156 if (!internal_iserror( 157 internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 158 reinterpret_cast<unsigned long>(&tag_bits), 159 reinterpret_cast<unsigned long>(&tag_shift), 0))) { 160 CHECK_EQ(tag_bits, kTagBits); 161 CHECK_EQ(tag_shift, kAddressTagShift); 162 return; 163 } 164 # endif // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) 165 if (flags()->fail_without_syscall_abi) { 166 Printf( 167 "FATAL: HWAddressSanitizer failed to enable tagged address syscall " 168 "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " 169 "configuration.\n"); 170 Die(); 171 } 172 } 173 #undef PR_SET_TAGGED_ADDR_CTRL 174 #undef PR_GET_TAGGED_ADDR_CTRL 175 #undef PR_TAGGED_ADDR_ENABLE 176 } 177 178 bool InitShadow() { 179 // Define the entire memory range. 180 kHighMemEnd = GetHighMemEnd(); 181 182 // Determine shadow memory base offset. 183 InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd)); 184 185 // Place the low memory first. 186 kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1; 187 kLowMemStart = 0; 188 189 // Define the low shadow based on the already placed low memory. 190 kLowShadowEnd = MemToShadow(kLowMemEnd); 191 kLowShadowStart = __hwasan_shadow_memory_dynamic_address; 192 193 // High shadow takes whatever memory is left up there (making sure it is not 194 // interfering with low memory in the fixed case). 195 kHighShadowEnd = MemToShadow(kHighMemEnd); 196 kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1; 197 198 // High memory starts where allocated shadow allows. 199 kHighMemStart = ShadowToMem(kHighShadowStart); 200 201 # if defined(HWASAN_ALIASING_MODE) 202 constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1); 203 kAliasRegionStart = 204 __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset; 205 206 CHECK_EQ(kAliasRegionStart >> kTaggableRegionCheckShift, 207 __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); 208 CHECK_EQ( 209 (kAliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift, 210 __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); 211 # endif 212 213 // Check the sanity of the defined memory ranges (there might be gaps). 214 CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0); 215 CHECK_GT(kHighMemStart, kHighShadowEnd); 216 CHECK_GT(kHighShadowEnd, kHighShadowStart); 217 CHECK_GT(kHighShadowStart, kLowMemEnd); 218 CHECK_GT(kLowMemEnd, kLowMemStart); 219 CHECK_GT(kLowShadowEnd, kLowShadowStart); 220 CHECK_GT(kLowShadowStart, kLowMemEnd); 221 222 if (Verbosity()) 223 PrintAddressSpaceLayout(); 224 225 // Reserve shadow memory. 226 ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow"); 227 ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow"); 228 229 // Protect all the gaps. 230 ProtectGap(0, Min(kLowMemStart, kLowShadowStart)); 231 if (kLowMemEnd + 1 < kLowShadowStart) 232 ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1); 233 if (kLowShadowEnd + 1 < kHighShadowStart) 234 ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1); 235 if (kHighShadowEnd + 1 < kHighMemStart) 236 ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1); 237 238 return true; 239 } 240 241 void InitThreads() { 242 CHECK(__hwasan_shadow_memory_dynamic_address); 243 uptr guard_page_size = GetMmapGranularity(); 244 uptr thread_space_start = 245 __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment); 246 uptr thread_space_end = 247 __hwasan_shadow_memory_dynamic_address - guard_page_size; 248 ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1, 249 "hwasan threads", /*madvise_shadow*/ false); 250 ProtectGap(thread_space_end, 251 __hwasan_shadow_memory_dynamic_address - thread_space_end); 252 InitThreadList(thread_space_start, thread_space_end - thread_space_start); 253 } 254 255 bool MemIsApp(uptr p) { 256 // Memory outside the alias range has non-zero tags. 257 # if !defined(HWASAN_ALIASING_MODE) 258 CHECK(GetTagFromPointer(p) == 0); 259 # endif 260 261 return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd); 262 } 263 264 void InstallAtExitHandler() { 265 atexit(HwasanAtExit); 266 } 267 268 // ---------------------- TSD ---------------- {{{1 269 270 extern "C" void __hwasan_thread_enter() { 271 hwasanThreadList().CreateCurrentThread()->InitRandomState(); 272 } 273 274 extern "C" void __hwasan_thread_exit() { 275 Thread *t = GetCurrentThread(); 276 // Make sure that signal handler can not see a stale current thread pointer. 277 atomic_signal_fence(memory_order_seq_cst); 278 if (t) 279 hwasanThreadList().ReleaseThread(t); 280 } 281 282 #if HWASAN_WITH_INTERCEPTORS 283 static pthread_key_t tsd_key; 284 static bool tsd_key_inited = false; 285 286 void HwasanTSDThreadInit() { 287 if (tsd_key_inited) 288 CHECK_EQ(0, pthread_setspecific(tsd_key, 289 (void *)GetPthreadDestructorIterations())); 290 } 291 292 void HwasanTSDDtor(void *tsd) { 293 uptr iterations = (uptr)tsd; 294 if (iterations > 1) { 295 CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1))); 296 return; 297 } 298 __hwasan_thread_exit(); 299 } 300 301 void HwasanTSDInit() { 302 CHECK(!tsd_key_inited); 303 tsd_key_inited = true; 304 CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor)); 305 } 306 #else 307 void HwasanTSDInit() {} 308 void HwasanTSDThreadInit() {} 309 #endif 310 311 #if SANITIZER_ANDROID 312 uptr *GetCurrentThreadLongPtr() { 313 return (uptr *)get_android_tls_ptr(); 314 } 315 #else 316 uptr *GetCurrentThreadLongPtr() { 317 return &__hwasan_tls; 318 } 319 #endif 320 321 #if SANITIZER_ANDROID 322 void AndroidTestTlsSlot() { 323 uptr kMagicValue = 0x010203040A0B0C0D; 324 uptr *tls_ptr = GetCurrentThreadLongPtr(); 325 uptr old_value = *tls_ptr; 326 *tls_ptr = kMagicValue; 327 dlerror(); 328 if (*(uptr *)get_android_tls_ptr() != kMagicValue) { 329 Printf( 330 "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used " 331 "for dlerror().\n"); 332 Die(); 333 } 334 *tls_ptr = old_value; 335 } 336 #else 337 void AndroidTestTlsSlot() {} 338 #endif 339 340 Thread *GetCurrentThread() { 341 uptr *ThreadLongPtr = GetCurrentThreadLongPtr(); 342 if (UNLIKELY(*ThreadLongPtr == 0)) 343 return nullptr; 344 auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr; 345 return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next()); 346 } 347 348 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { 349 // Access type is passed in a platform dependent way (see below) and encoded 350 // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is 351 // recoverable. Valid values of Y are 0 to 4, which are interpreted as 352 // log2(access_size), and 0xF, which means that access size is passed via 353 // platform dependent register (see below). 354 #if defined(__aarch64__) 355 // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF, 356 // access size is stored in X1 register. Access address is always in X0 357 // register. 358 uptr pc = (uptr)info->si_addr; 359 const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff; 360 if ((code & 0xff00) != 0x900) 361 return AccessInfo{}; // Not ours. 362 363 const bool is_store = code & 0x10; 364 const bool recover = code & 0x20; 365 const uptr addr = uc->uc_mcontext.regs[0]; 366 const unsigned size_log = code & 0xf; 367 if (size_log > 4 && size_log != 0xf) 368 return AccessInfo{}; // Not ours. 369 const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log; 370 371 #elif defined(__x86_64__) 372 // Access type is encoded in the instruction following INT3 as 373 // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in 374 // RSI register. Access address is always in RDI register. 375 uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP]; 376 uint8_t *nop = (uint8_t*)pc; 377 if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 || 378 *(nop + 3) < 0x40) 379 return AccessInfo{}; // Not ours. 380 const unsigned code = *(nop + 3); 381 382 const bool is_store = code & 0x10; 383 const bool recover = code & 0x20; 384 const uptr addr = uc->uc_mcontext.gregs[REG_RDI]; 385 const unsigned size_log = code & 0xf; 386 if (size_log > 4 && size_log != 0xf) 387 return AccessInfo{}; // Not ours. 388 const uptr size = 389 size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log; 390 391 #else 392 # error Unsupported architecture 393 #endif 394 395 return AccessInfo{addr, size, is_store, !is_store, recover}; 396 } 397 398 static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { 399 AccessInfo ai = GetAccessInfo(info, uc); 400 if (!ai.is_store && !ai.is_load) 401 return false; 402 403 SignalContext sig{info, uc}; 404 HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc); 405 406 #if defined(__aarch64__) 407 uc->uc_mcontext.pc += 4; 408 #elif defined(__x86_64__) 409 #else 410 # error Unsupported architecture 411 #endif 412 return true; 413 } 414 415 static void OnStackUnwind(const SignalContext &sig, const void *, 416 BufferedStackTrace *stack) { 417 stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context, 418 common_flags()->fast_unwind_on_fatal); 419 } 420 421 void HwasanOnDeadlySignal(int signo, void *info, void *context) { 422 // Probably a tag mismatch. 423 if (signo == SIGTRAP) 424 if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t*)context)) 425 return; 426 427 HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr); 428 } 429 430 431 } // namespace __hwasan 432 433 // Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the 434 // rest of the mismatch handling code (C++). 435 void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame, 436 size_t outsize) { 437 __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize); 438 } 439 440 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD 441