1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file is a part of HWAddressSanitizer, an address sanity checker
11 /// based on tagged addressing.
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15 #include "llvm/ADT/MapVector.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/StringExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/BinaryFormat/ELF.h"
21 #include "llvm/IR/Attributes.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/Constant.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DebugInfoMetadata.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/IRBuilder.h"
30 #include "llvm/IR/InlineAsm.h"
31 #include "llvm/IR/InstVisitor.h"
32 #include "llvm/IR/Instruction.h"
33 #include "llvm/IR/Instructions.h"
34 #include "llvm/IR/IntrinsicInst.h"
35 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/IR/LLVMContext.h"
37 #include "llvm/IR/MDBuilder.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/IR/Value.h"
41 #include "llvm/InitializePasses.h"
42 #include "llvm/Pass.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/CommandLine.h"
45 #include "llvm/Support/Debug.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Transforms/Instrumentation.h"
48 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
49 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
50 #include "llvm/Transforms/Utils/ModuleUtils.h"
51 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
52 #include <sstream>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "hwasan"
57 
58 const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
59 const char kHwasanNoteName[] = "hwasan.note";
60 const char kHwasanInitName[] = "__hwasan_init";
61 const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
62 
63 const char kHwasanShadowMemoryDynamicAddress[] =
64     "__hwasan_shadow_memory_dynamic_address";
65 
66 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
67 static const size_t kNumberOfAccessSizes = 5;
68 
69 static const size_t kDefaultShadowScale = 4;
70 static const uint64_t kDynamicShadowSentinel =
71     std::numeric_limits<uint64_t>::max();
72 static const unsigned kPointerTagShift = 56;
73 
74 static const unsigned kShadowBaseAlignment = 32;
75 
76 static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
77     "hwasan-memory-access-callback-prefix",
78     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
79     cl::init("__hwasan_"));
80 
81 static cl::opt<bool>
82     ClInstrumentWithCalls("hwasan-instrument-with-calls",
83                 cl::desc("instrument reads and writes with callbacks"),
84                 cl::Hidden, cl::init(false));
85 
86 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
87                                        cl::desc("instrument read instructions"),
88                                        cl::Hidden, cl::init(true));
89 
90 static cl::opt<bool> ClInstrumentWrites(
91     "hwasan-instrument-writes", cl::desc("instrument write instructions"),
92     cl::Hidden, cl::init(true));
93 
94 static cl::opt<bool> ClInstrumentAtomics(
95     "hwasan-instrument-atomics",
96     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
97     cl::init(true));
98 
99 static cl::opt<bool> ClInstrumentByval("hwasan-instrument-byval",
100                                        cl::desc("instrument byval arguments"),
101                                        cl::Hidden, cl::init(true));
102 
103 static cl::opt<bool> ClRecover(
104     "hwasan-recover",
105     cl::desc("Enable recovery mode (continue-after-error)."),
106     cl::Hidden, cl::init(false));
107 
108 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
109                                        cl::desc("instrument stack (allocas)"),
110                                        cl::Hidden, cl::init(true));
111 
112 static cl::opt<bool> ClUARRetagToZero(
113     "hwasan-uar-retag-to-zero",
114     cl::desc("Clear alloca tags before returning from the function to allow "
115              "non-instrumented and instrumented function calls mix. When set "
116              "to false, allocas are retagged before returning from the "
117              "function to detect use after return."),
118     cl::Hidden, cl::init(true));
119 
120 static cl::opt<bool> ClGenerateTagsWithCalls(
121     "hwasan-generate-tags-with-calls",
122     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
123     cl::init(false));
124 
125 static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
126                                cl::Hidden, cl::init(false), cl::ZeroOrMore);
127 
128 static cl::opt<int> ClMatchAllTag(
129     "hwasan-match-all-tag",
130     cl::desc("don't report bad accesses via pointers with this tag"),
131     cl::Hidden, cl::init(-1));
132 
133 static cl::opt<bool> ClEnableKhwasan(
134     "hwasan-kernel",
135     cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
136     cl::Hidden, cl::init(false));
137 
138 // These flags allow to change the shadow mapping and control how shadow memory
139 // is accessed. The shadow mapping looks like:
140 //    Shadow = (Mem >> scale) + offset
141 
142 static cl::opt<uint64_t>
143     ClMappingOffset("hwasan-mapping-offset",
144                     cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
145                     cl::Hidden, cl::init(0));
146 
147 static cl::opt<bool>
148     ClWithIfunc("hwasan-with-ifunc",
149                 cl::desc("Access dynamic shadow through an ifunc global on "
150                          "platforms that support this"),
151                 cl::Hidden, cl::init(false));
152 
153 static cl::opt<bool> ClWithTls(
154     "hwasan-with-tls",
155     cl::desc("Access dynamic shadow through an thread-local pointer on "
156              "platforms that support this"),
157     cl::Hidden, cl::init(true));
158 
159 static cl::opt<bool>
160     ClRecordStackHistory("hwasan-record-stack-history",
161                          cl::desc("Record stack frames with tagged allocations "
162                                   "in a thread-local ring buffer"),
163                          cl::Hidden, cl::init(true));
164 static cl::opt<bool>
165     ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
166                               cl::desc("instrument memory intrinsics"),
167                               cl::Hidden, cl::init(true));
168 
169 static cl::opt<bool>
170     ClInstrumentLandingPads("hwasan-instrument-landing-pads",
171                             cl::desc("instrument landing pads"), cl::Hidden,
172                             cl::init(false), cl::ZeroOrMore);
173 
174 static cl::opt<bool> ClUseShortGranules(
175     "hwasan-use-short-granules",
176     cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
177     cl::init(false), cl::ZeroOrMore);
178 
179 static cl::opt<bool> ClInstrumentPersonalityFunctions(
180     "hwasan-instrument-personality-functions",
181     cl::desc("instrument personality functions"), cl::Hidden, cl::init(false),
182     cl::ZeroOrMore);
183 
184 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
185                                        cl::desc("inline all checks"),
186                                        cl::Hidden, cl::init(false));
187 
188 namespace {
189 
190 /// An instrumentation pass implementing detection of addressability bugs
191 /// using tagged pointers.
192 class HWAddressSanitizer {
193 public:
194   explicit HWAddressSanitizer(Module &M, bool CompileKernel = false,
195                               bool Recover = false) : M(M) {
196     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
197     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
198         ClEnableKhwasan : CompileKernel;
199 
200     initializeModule();
201   }
202 
203   bool sanitizeFunction(Function &F);
204   void initializeModule();
205   void createHwasanCtorComdat();
206 
207   void initializeCallbacks(Module &M);
208 
209   Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
210 
211   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
212   Value *getShadowNonTls(IRBuilder<> &IRB);
213 
214   void untagPointerOperand(Instruction *I, Value *Addr);
215   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
216   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
217                                  unsigned AccessSizeIndex,
218                                  Instruction *InsertBefore);
219   void instrumentMemIntrinsic(MemIntrinsic *MI);
220   bool instrumentMemAccess(InterestingMemoryOperand &O);
221   bool ignoreAccess(Value *Ptr);
222   void getInterestingMemoryOperands(
223       Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
224 
225   bool isInterestingAlloca(const AllocaInst &AI);
226   bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
227   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
228   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
229   bool instrumentStack(
230       SmallVectorImpl<AllocaInst *> &Allocas,
231       DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap,
232       SmallVectorImpl<Instruction *> &RetVec, Value *StackTag);
233   Value *readRegister(IRBuilder<> &IRB, StringRef Name);
234   bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
235   Value *getNextTagWithCall(IRBuilder<> &IRB);
236   Value *getStackBaseTag(IRBuilder<> &IRB);
237   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
238                      unsigned AllocaNo);
239   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
240 
241   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
242   void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
243 
244   void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
245   void instrumentGlobals();
246 
247   void instrumentPersonalityFunctions();
248 
249 private:
250   LLVMContext *C;
251   Module &M;
252   Triple TargetTriple;
253   FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
254   FunctionCallee HWAsanHandleVfork;
255 
256   /// This struct defines the shadow mapping using the rule:
257   ///   shadow = (mem >> Scale) + Offset.
258   /// If InGlobal is true, then
259   ///   extern char __hwasan_shadow[];
260   ///   shadow = (mem >> Scale) + &__hwasan_shadow
261   /// If InTls is true, then
262   ///   extern char *__hwasan_tls;
263   ///   shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
264   struct ShadowMapping {
265     int Scale;
266     uint64_t Offset;
267     bool InGlobal;
268     bool InTls;
269 
270     void init(Triple &TargetTriple, bool InstrumentWithCalls);
271     unsigned getObjectAlignment() const { return 1U << Scale; }
272   };
273   ShadowMapping Mapping;
274 
275   Type *VoidTy = Type::getVoidTy(M.getContext());
276   Type *IntptrTy;
277   Type *Int8PtrTy;
278   Type *Int8Ty;
279   Type *Int32Ty;
280   Type *Int64Ty = Type::getInt64Ty(M.getContext());
281 
282   bool CompileKernel;
283   bool Recover;
284   bool OutlinedChecks;
285   bool UseShortGranules;
286   bool InstrumentLandingPads;
287   bool InstrumentWithCalls;
288   bool InstrumentStack;
289   bool UsePageAliases;
290 
291   bool HasMatchAllTag = false;
292   uint8_t MatchAllTag = 0;
293 
294   Function *HwasanCtorFunction;
295 
296   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
297   FunctionCallee HwasanMemoryAccessCallbackSized[2];
298 
299   FunctionCallee HwasanTagMemoryFunc;
300   FunctionCallee HwasanGenerateTagFunc;
301 
302   Constant *ShadowGlobal;
303 
304   Value *ShadowBase = nullptr;
305   Value *StackBaseTag = nullptr;
306   GlobalValue *ThreadPtrGlobal = nullptr;
307 };
308 
309 class HWAddressSanitizerLegacyPass : public FunctionPass {
310 public:
311   // Pass identification, replacement for typeid.
312   static char ID;
313 
314   explicit HWAddressSanitizerLegacyPass(bool CompileKernel = false,
315                                         bool Recover = false)
316       : FunctionPass(ID), CompileKernel(CompileKernel), Recover(Recover) {
317     initializeHWAddressSanitizerLegacyPassPass(
318         *PassRegistry::getPassRegistry());
319   }
320 
321   StringRef getPassName() const override { return "HWAddressSanitizer"; }
322 
323   bool doInitialization(Module &M) override {
324     HWASan = std::make_unique<HWAddressSanitizer>(M, CompileKernel, Recover);
325     return true;
326   }
327 
328   bool runOnFunction(Function &F) override {
329     return HWASan->sanitizeFunction(F);
330   }
331 
332   bool doFinalization(Module &M) override {
333     HWASan.reset();
334     return false;
335   }
336 
337 private:
338   std::unique_ptr<HWAddressSanitizer> HWASan;
339   bool CompileKernel;
340   bool Recover;
341 };
342 
343 } // end anonymous namespace
344 
345 char HWAddressSanitizerLegacyPass::ID = 0;
346 
347 INITIALIZE_PASS_BEGIN(
348     HWAddressSanitizerLegacyPass, "hwasan",
349     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
350     false)
351 INITIALIZE_PASS_END(
352     HWAddressSanitizerLegacyPass, "hwasan",
353     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
354     false)
355 
356 FunctionPass *llvm::createHWAddressSanitizerLegacyPassPass(bool CompileKernel,
357                                                            bool Recover) {
358   assert(!CompileKernel || Recover);
359   return new HWAddressSanitizerLegacyPass(CompileKernel, Recover);
360 }
361 
362 HWAddressSanitizerPass::HWAddressSanitizerPass(bool CompileKernel, bool Recover)
363     : CompileKernel(CompileKernel), Recover(Recover) {}
364 
365 PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
366                                               ModuleAnalysisManager &MAM) {
367   HWAddressSanitizer HWASan(M, CompileKernel, Recover);
368   bool Modified = false;
369   for (Function &F : M)
370     Modified |= HWASan.sanitizeFunction(F);
371   if (Modified)
372     return PreservedAnalyses::none();
373   return PreservedAnalyses::all();
374 }
375 
376 void HWAddressSanitizer::createHwasanCtorComdat() {
377   std::tie(HwasanCtorFunction, std::ignore) =
378       getOrCreateSanitizerCtorAndInitFunctions(
379           M, kHwasanModuleCtorName, kHwasanInitName,
380           /*InitArgTypes=*/{},
381           /*InitArgs=*/{},
382           // This callback is invoked when the functions are created the first
383           // time. Hook them into the global ctors list in that case:
384           [&](Function *Ctor, FunctionCallee) {
385             Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
386             Ctor->setComdat(CtorComdat);
387             appendToGlobalCtors(M, Ctor, 0, Ctor);
388           });
389 
390   // Create a note that contains pointers to the list of global
391   // descriptors. Adding a note to the output file will cause the linker to
392   // create a PT_NOTE program header pointing to the note that we can use to
393   // find the descriptor list starting from the program headers. A function
394   // provided by the runtime initializes the shadow memory for the globals by
395   // accessing the descriptor list via the note. The dynamic loader needs to
396   // call this function whenever a library is loaded.
397   //
398   // The reason why we use a note for this instead of a more conventional
399   // approach of having a global constructor pass a descriptor list pointer to
400   // the runtime is because of an order of initialization problem. With
401   // constructors we can encounter the following problematic scenario:
402   //
403   // 1) library A depends on library B and also interposes one of B's symbols
404   // 2) B's constructors are called before A's (as required for correctness)
405   // 3) during construction, B accesses one of its "own" globals (actually
406   //    interposed by A) and triggers a HWASAN failure due to the initialization
407   //    for A not having happened yet
408   //
409   // Even without interposition it is possible to run into similar situations in
410   // cases where two libraries mutually depend on each other.
411   //
412   // We only need one note per binary, so put everything for the note in a
413   // comdat. This needs to be a comdat with an .init_array section to prevent
414   // newer versions of lld from discarding the note.
415   //
416   // Create the note even if we aren't instrumenting globals. This ensures that
417   // binaries linked from object files with both instrumented and
418   // non-instrumented globals will end up with a note, even if a comdat from an
419   // object file with non-instrumented globals is selected. The note is harmless
420   // if the runtime doesn't support it, since it will just be ignored.
421   Comdat *NoteComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
422 
423   Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
424   auto Start =
425       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
426                          nullptr, "__start_hwasan_globals");
427   Start->setVisibility(GlobalValue::HiddenVisibility);
428   Start->setDSOLocal(true);
429   auto Stop =
430       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
431                          nullptr, "__stop_hwasan_globals");
432   Stop->setVisibility(GlobalValue::HiddenVisibility);
433   Stop->setDSOLocal(true);
434 
435   // Null-terminated so actually 8 bytes, which are required in order to align
436   // the note properly.
437   auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
438 
439   auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
440                                  Int32Ty, Int32Ty);
441   auto *Note =
442       new GlobalVariable(M, NoteTy, /*isConstant=*/true,
443                          GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
444   Note->setSection(".note.hwasan.globals");
445   Note->setComdat(NoteComdat);
446   Note->setAlignment(Align(4));
447   Note->setDSOLocal(true);
448 
449   // The pointers in the note need to be relative so that the note ends up being
450   // placed in rodata, which is the standard location for notes.
451   auto CreateRelPtr = [&](Constant *Ptr) {
452     return ConstantExpr::getTrunc(
453         ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
454                              ConstantExpr::getPtrToInt(Note, Int64Ty)),
455         Int32Ty);
456   };
457   Note->setInitializer(ConstantStruct::getAnon(
458       {ConstantInt::get(Int32Ty, 8),                           // n_namesz
459        ConstantInt::get(Int32Ty, 8),                           // n_descsz
460        ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
461        Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
462   appendToCompilerUsed(M, Note);
463 
464   // Create a zero-length global in hwasan_globals so that the linker will
465   // always create start and stop symbols.
466   auto Dummy = new GlobalVariable(
467       M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
468       Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
469   Dummy->setSection("hwasan_globals");
470   Dummy->setComdat(NoteComdat);
471   Dummy->setMetadata(LLVMContext::MD_associated,
472                      MDNode::get(*C, ValueAsMetadata::get(Note)));
473   appendToCompilerUsed(M, Dummy);
474 }
475 
476 /// Module-level initialization.
477 ///
478 /// inserts a call to __hwasan_init to the module's constructor list.
479 void HWAddressSanitizer::initializeModule() {
480   LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
481   auto &DL = M.getDataLayout();
482 
483   TargetTriple = Triple(M.getTargetTriple());
484 
485   // x86_64 uses userspace pointer aliases, currently heap-only with callback
486   // instrumentation only.
487   UsePageAliases = TargetTriple.getArch() == Triple::x86_64;
488   InstrumentWithCalls = UsePageAliases ? true : ClInstrumentWithCalls;
489   InstrumentStack = UsePageAliases ? false : ClInstrumentStack;
490 
491   Mapping.init(TargetTriple, InstrumentWithCalls);
492 
493   C = &(M.getContext());
494   IRBuilder<> IRB(*C);
495   IntptrTy = IRB.getIntPtrTy(DL);
496   Int8PtrTy = IRB.getInt8PtrTy();
497   Int8Ty = IRB.getInt8Ty();
498   Int32Ty = IRB.getInt32Ty();
499 
500   HwasanCtorFunction = nullptr;
501 
502   // Older versions of Android do not have the required runtime support for
503   // short granules, global or personality function instrumentation. On other
504   // platforms we currently require using the latest version of the runtime.
505   bool NewRuntime =
506       !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
507 
508   UseShortGranules =
509       ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
510   OutlinedChecks =
511       TargetTriple.isAArch64() && TargetTriple.isOSBinFormatELF() &&
512       (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
513 
514   if (ClMatchAllTag.getNumOccurrences()) {
515     if (ClMatchAllTag != -1) {
516       HasMatchAllTag = true;
517       MatchAllTag = ClMatchAllTag & 0xFF;
518     }
519   } else if (CompileKernel) {
520     HasMatchAllTag = true;
521     MatchAllTag = 0xFF;
522   }
523 
524   // If we don't have personality function support, fall back to landing pads.
525   InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
526                               ? ClInstrumentLandingPads
527                               : !NewRuntime;
528 
529   if (!CompileKernel) {
530     createHwasanCtorComdat();
531     bool InstrumentGlobals =
532         ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
533     if (InstrumentGlobals && !UsePageAliases)
534       instrumentGlobals();
535 
536     bool InstrumentPersonalityFunctions =
537         ClInstrumentPersonalityFunctions.getNumOccurrences()
538             ? ClInstrumentPersonalityFunctions
539             : NewRuntime;
540     if (InstrumentPersonalityFunctions)
541       instrumentPersonalityFunctions();
542   }
543 
544   if (!TargetTriple.isAndroid()) {
545     Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
546       auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
547                                     GlobalValue::ExternalLinkage, nullptr,
548                                     "__hwasan_tls", nullptr,
549                                     GlobalVariable::InitialExecTLSModel);
550       appendToCompilerUsed(M, GV);
551       return GV;
552     });
553     ThreadPtrGlobal = cast<GlobalVariable>(C);
554   }
555 }
556 
557 void HWAddressSanitizer::initializeCallbacks(Module &M) {
558   IRBuilder<> IRB(*C);
559   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
560     const std::string TypeStr = AccessIsWrite ? "store" : "load";
561     const std::string EndingStr = Recover ? "_noabort" : "";
562 
563     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
564         ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
565         FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
566 
567     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
568          AccessSizeIndex++) {
569       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
570           M.getOrInsertFunction(
571               ClMemoryAccessCallbackPrefix + TypeStr +
572                   itostr(1ULL << AccessSizeIndex) + EndingStr,
573               FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
574     }
575   }
576 
577   HwasanTagMemoryFunc = M.getOrInsertFunction(
578       "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
579   HwasanGenerateTagFunc =
580       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
581 
582   ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
583                                      ArrayType::get(IRB.getInt8Ty(), 0));
584 
585   const std::string MemIntrinCallbackPrefix =
586       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
587   HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
588                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
589                                         IRB.getInt8PtrTy(), IntptrTy);
590   HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
591                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
592                                        IRB.getInt8PtrTy(), IntptrTy);
593   HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
594                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
595                                        IRB.getInt32Ty(), IntptrTy);
596 
597   HWAsanHandleVfork =
598       M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy);
599 }
600 
601 Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
602   // An empty inline asm with input reg == output reg.
603   // An opaque no-op cast, basically.
604   // This prevents code bloat as a result of rematerializing trivial definitions
605   // such as constants or global addresses at every load and store.
606   InlineAsm *Asm =
607       InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
608                      StringRef(""), StringRef("=r,0"),
609                      /*hasSideEffects=*/false);
610   return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
611 }
612 
613 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
614   return getOpaqueNoopCast(IRB, ShadowGlobal);
615 }
616 
617 Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
618   if (Mapping.Offset != kDynamicShadowSentinel)
619     return getOpaqueNoopCast(
620         IRB, ConstantExpr::getIntToPtr(
621                  ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy));
622 
623   if (Mapping.InGlobal) {
624     return getDynamicShadowIfunc(IRB);
625   } else {
626     Value *GlobalDynamicAddress =
627         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
628             kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
629     return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
630   }
631 }
632 
633 bool HWAddressSanitizer::ignoreAccess(Value *Ptr) {
634   // Do not instrument acesses from different address spaces; we cannot deal
635   // with them.
636   Type *PtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
637   if (PtrTy->getPointerAddressSpace() != 0)
638     return true;
639 
640   // Ignore swifterror addresses.
641   // swifterror memory addresses are mem2reg promoted by instruction
642   // selection. As such they cannot have regular uses like an instrumentation
643   // function and it makes no sense to track them as memory.
644   if (Ptr->isSwiftError())
645     return true;
646 
647   return false;
648 }
649 
650 void HWAddressSanitizer::getInterestingMemoryOperands(
651     Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
652   // Skip memory accesses inserted by another instrumentation.
653   if (I->hasMetadata("nosanitize"))
654     return;
655 
656   // Do not instrument the load fetching the dynamic shadow address.
657   if (ShadowBase == I)
658     return;
659 
660   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
661     if (!ClInstrumentReads || ignoreAccess(LI->getPointerOperand()))
662       return;
663     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
664                              LI->getType(), LI->getAlign());
665   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
666     if (!ClInstrumentWrites || ignoreAccess(SI->getPointerOperand()))
667       return;
668     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
669                              SI->getValueOperand()->getType(), SI->getAlign());
670   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
671     if (!ClInstrumentAtomics || ignoreAccess(RMW->getPointerOperand()))
672       return;
673     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
674                              RMW->getValOperand()->getType(), None);
675   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
676     if (!ClInstrumentAtomics || ignoreAccess(XCHG->getPointerOperand()))
677       return;
678     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
679                              XCHG->getCompareOperand()->getType(), None);
680   } else if (auto CI = dyn_cast<CallInst>(I)) {
681     for (unsigned ArgNo = 0; ArgNo < CI->getNumArgOperands(); ArgNo++) {
682       if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
683           ignoreAccess(CI->getArgOperand(ArgNo)))
684         continue;
685       Type *Ty = CI->getParamByValType(ArgNo);
686       Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
687     }
688   }
689 }
690 
691 static unsigned getPointerOperandIndex(Instruction *I) {
692   if (LoadInst *LI = dyn_cast<LoadInst>(I))
693     return LI->getPointerOperandIndex();
694   if (StoreInst *SI = dyn_cast<StoreInst>(I))
695     return SI->getPointerOperandIndex();
696   if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
697     return RMW->getPointerOperandIndex();
698   if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
699     return XCHG->getPointerOperandIndex();
700   report_fatal_error("Unexpected instruction");
701   return -1;
702 }
703 
704 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
705   size_t Res = countTrailingZeros(TypeSize / 8);
706   assert(Res < kNumberOfAccessSizes);
707   return Res;
708 }
709 
710 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
711   if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64)
712     return;
713 
714   IRBuilder<> IRB(I);
715   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
716   Value *UntaggedPtr =
717       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
718   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
719 }
720 
721 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
722   // Mem >> Scale
723   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
724   if (Mapping.Offset == 0)
725     return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
726   // (Mem >> Scale) + Offset
727   return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow);
728 }
729 
730 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
731                                                    unsigned AccessSizeIndex,
732                                                    Instruction *InsertBefore) {
733   assert(!UsePageAliases);
734   const int64_t AccessInfo =
735       (CompileKernel << HWASanAccessInfo::CompileKernelShift) +
736       (HasMatchAllTag << HWASanAccessInfo::HasMatchAllShift) +
737       (MatchAllTag << HWASanAccessInfo::MatchAllShift) +
738       (Recover << HWASanAccessInfo::RecoverShift) +
739       (IsWrite << HWASanAccessInfo::IsWriteShift) +
740       (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
741   IRBuilder<> IRB(InsertBefore);
742 
743   if (OutlinedChecks) {
744     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
745     Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
746     IRB.CreateCall(Intrinsic::getDeclaration(
747                        M, UseShortGranules
748                               ? Intrinsic::hwasan_check_memaccess_shortgranules
749                               : Intrinsic::hwasan_check_memaccess),
750                    {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
751     return;
752   }
753 
754   Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
755   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
756                                   IRB.getInt8Ty());
757   Value *AddrLong = untagPointer(IRB, PtrLong);
758   Value *Shadow = memToShadow(AddrLong, IRB);
759   Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
760   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
761 
762   if (HasMatchAllTag) {
763     Value *TagNotIgnored = IRB.CreateICmpNE(
764         PtrTag, ConstantInt::get(PtrTag->getType(), MatchAllTag));
765     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
766   }
767 
768   Instruction *CheckTerm =
769       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
770                                 MDBuilder(*C).createBranchWeights(1, 100000));
771 
772   IRB.SetInsertPoint(CheckTerm);
773   Value *OutOfShortGranuleTagRange =
774       IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15));
775   Instruction *CheckFailTerm =
776       SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover,
777                                 MDBuilder(*C).createBranchWeights(1, 100000));
778 
779   IRB.SetInsertPoint(CheckTerm);
780   Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty);
781   PtrLowBits = IRB.CreateAdd(
782       PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1));
783   Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag);
784   SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false,
785                             MDBuilder(*C).createBranchWeights(1, 100000),
786                             (DomTreeUpdater *)nullptr, nullptr,
787                             CheckFailTerm->getParent());
788 
789   IRB.SetInsertPoint(CheckTerm);
790   Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15);
791   InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy);
792   Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);
793   Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag);
794   SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false,
795                             MDBuilder(*C).createBranchWeights(1, 100000),
796                             (DomTreeUpdater *)nullptr, nullptr,
797                             CheckFailTerm->getParent());
798 
799   IRB.SetInsertPoint(CheckFailTerm);
800   InlineAsm *Asm;
801   switch (TargetTriple.getArch()) {
802     case Triple::x86_64:
803       // The signal handler will find the data address in rdi.
804       Asm = InlineAsm::get(
805           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
806           "int3\nnopl " +
807               itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
808               "(%rax)",
809           "{rdi}",
810           /*hasSideEffects=*/true);
811       break;
812     case Triple::aarch64:
813     case Triple::aarch64_be:
814       // The signal handler will find the data address in x0.
815       Asm = InlineAsm::get(
816           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
817           "brk #" +
818               itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
819           "{x0}",
820           /*hasSideEffects=*/true);
821       break;
822     default:
823       report_fatal_error("unsupported architecture");
824   }
825   IRB.CreateCall(Asm, PtrLong);
826   if (Recover)
827     cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent());
828 }
829 
830 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
831   IRBuilder<> IRB(MI);
832   if (isa<MemTransferInst>(MI)) {
833     IRB.CreateCall(
834         isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
835         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
836          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
837          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
838   } else if (isa<MemSetInst>(MI)) {
839     IRB.CreateCall(
840         HWAsanMemset,
841         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
842          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
843          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
844   }
845   MI->eraseFromParent();
846 }
847 
848 bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
849   Value *Addr = O.getPtr();
850 
851   LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
852 
853   if (O.MaybeMask)
854     return false; //FIXME
855 
856   IRBuilder<> IRB(O.getInsn());
857   if (isPowerOf2_64(O.TypeSize) &&
858       (O.TypeSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
859       (!O.Alignment || *O.Alignment >= (1ULL << Mapping.Scale) ||
860        *O.Alignment >= O.TypeSize / 8)) {
861     size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeSize);
862     if (InstrumentWithCalls) {
863       IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
864                      IRB.CreatePointerCast(Addr, IntptrTy));
865     } else {
866       instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
867     }
868   } else {
869     IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite],
870                    {IRB.CreatePointerCast(Addr, IntptrTy),
871                     ConstantInt::get(IntptrTy, O.TypeSize / 8)});
872   }
873   untagPointerOperand(O.getInsn(), Addr);
874 
875   return true;
876 }
877 
878 static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
879   uint64_t ArraySize = 1;
880   if (AI.isArrayAllocation()) {
881     const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize());
882     assert(CI && "non-constant array size");
883     ArraySize = CI->getZExtValue();
884   }
885   Type *Ty = AI.getAllocatedType();
886   uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
887   return SizeInBytes * ArraySize;
888 }
889 
890 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
891                                    Value *Tag, size_t Size) {
892   size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
893   if (!UseShortGranules)
894     Size = AlignedSize;
895 
896   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
897   if (InstrumentWithCalls) {
898     IRB.CreateCall(HwasanTagMemoryFunc,
899                    {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
900                     ConstantInt::get(IntptrTy, AlignedSize)});
901   } else {
902     size_t ShadowSize = Size >> Mapping.Scale;
903     Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
904     // If this memset is not inlined, it will be intercepted in the hwasan
905     // runtime library. That's OK, because the interceptor skips the checks if
906     // the address is in the shadow region.
907     // FIXME: the interceptor is not as fast as real memset. Consider lowering
908     // llvm.memset right here into either a sequence of stores, or a call to
909     // hwasan_tag_memory.
910     if (ShadowSize)
911       IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, Align(1));
912     if (Size != AlignedSize) {
913       IRB.CreateStore(
914           ConstantInt::get(Int8Ty, Size % Mapping.getObjectAlignment()),
915           IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
916       IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32(
917                                    Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy),
918                                    AlignedSize - 1));
919     }
920   }
921   return true;
922 }
923 
924 static unsigned RetagMask(unsigned AllocaNo) {
925   // A list of 8-bit numbers that have at most one run of non-zero bits.
926   // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
927   // masks.
928   // The list does not include the value 255, which is used for UAR.
929   //
930   // Because we are more likely to use earlier elements of this list than later
931   // ones, it is sorted in increasing order of probability of collision with a
932   // mask allocated (temporally) nearby. The program that generated this list
933   // can be found at:
934   // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
935   static unsigned FastMasks[] = {0,  128, 64,  192, 32,  96,  224, 112, 240,
936                                  48, 16,  120, 248, 56,  24,  8,   124, 252,
937                                  60, 28,  12,  4,   126, 254, 62,  30,  14,
938                                  6,  2,   127, 63,  31,  15,  7,   3,   1};
939   return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))];
940 }
941 
942 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
943   return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
944 }
945 
946 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
947   if (ClGenerateTagsWithCalls)
948     return getNextTagWithCall(IRB);
949   if (StackBaseTag)
950     return StackBaseTag;
951   // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
952   // first).
953   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
954   auto GetStackPointerFn = Intrinsic::getDeclaration(
955       M, Intrinsic::frameaddress,
956       IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
957   Value *StackPointer = IRB.CreateCall(
958       GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())});
959 
960   // Extract some entropy from the stack pointer for the tags.
961   // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
962   // between functions).
963   Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy);
964   Value *StackTag =
965       IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20),
966                     "hwasan.stack.base.tag");
967   return StackTag;
968 }
969 
970 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
971                                         AllocaInst *AI, unsigned AllocaNo) {
972   if (ClGenerateTagsWithCalls)
973     return getNextTagWithCall(IRB);
974   return IRB.CreateXor(StackTag,
975                        ConstantInt::get(IntptrTy, RetagMask(AllocaNo)));
976 }
977 
978 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
979   if (ClUARRetagToZero)
980     return ConstantInt::get(IntptrTy, 0);
981   if (ClGenerateTagsWithCalls)
982     return getNextTagWithCall(IRB);
983   return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
984 }
985 
986 // Add a tag to an address.
987 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
988                                       Value *PtrLong, Value *Tag) {
989   assert(!UsePageAliases);
990   Value *TaggedPtrLong;
991   if (CompileKernel) {
992     // Kernel addresses have 0xFF in the most significant byte.
993     Value *ShiftedTag = IRB.CreateOr(
994         IRB.CreateShl(Tag, kPointerTagShift),
995         ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
996     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
997   } else {
998     // Userspace can simply do OR (tag << 56);
999     Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
1000     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
1001   }
1002   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
1003 }
1004 
1005 // Remove tag from an address.
1006 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
1007   assert(!UsePageAliases);
1008   Value *UntaggedPtrLong;
1009   if (CompileKernel) {
1010     // Kernel addresses have 0xFF in the most significant byte.
1011     UntaggedPtrLong = IRB.CreateOr(PtrLong,
1012         ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
1013   } else {
1014     // Userspace addresses have 0x00.
1015     UntaggedPtrLong = IRB.CreateAnd(PtrLong,
1016         ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
1017   }
1018   return UntaggedPtrLong;
1019 }
1020 
1021 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
1022   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1023   if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
1024     // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1025     // in Bionic's libc/private/bionic_tls.h.
1026     Function *ThreadPointerFunc =
1027         Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
1028     Value *SlotPtr = IRB.CreatePointerCast(
1029         IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
1030                                IRB.CreateCall(ThreadPointerFunc), 0x30),
1031         Ty->getPointerTo(0));
1032     return SlotPtr;
1033   }
1034   if (ThreadPtrGlobal)
1035     return ThreadPtrGlobal;
1036 
1037 
1038   return nullptr;
1039 }
1040 
1041 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1042   if (!Mapping.InTls) {
1043     ShadowBase = getShadowNonTls(IRB);
1044     return;
1045   }
1046 
1047   if (!WithFrameRecord && TargetTriple.isAndroid()) {
1048     ShadowBase = getDynamicShadowIfunc(IRB);
1049     return;
1050   }
1051 
1052   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
1053   assert(SlotPtr);
1054 
1055   Value *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
1056   // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
1057   Value *ThreadLongMaybeUntagged =
1058       TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong);
1059 
1060   if (WithFrameRecord) {
1061     Function *F = IRB.GetInsertBlock()->getParent();
1062     StackBaseTag = IRB.CreateAShr(ThreadLong, 3);
1063 
1064     // Prepare ring buffer data.
1065     Value *PC;
1066     if (TargetTriple.getArch() == Triple::aarch64)
1067       PC = readRegister(IRB, "pc");
1068     else
1069       PC = IRB.CreatePtrToInt(F, IntptrTy);
1070     Module *M = F->getParent();
1071     auto GetStackPointerFn = Intrinsic::getDeclaration(
1072         M, Intrinsic::frameaddress,
1073         IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
1074     Value *SP = IRB.CreatePtrToInt(
1075         IRB.CreateCall(GetStackPointerFn,
1076                        {Constant::getNullValue(IRB.getInt32Ty())}),
1077         IntptrTy);
1078     // Mix SP and PC.
1079     // Assumptions:
1080     // PC is 0x0000PPPPPPPPPPPP  (48 bits are meaningful, others are zero)
1081     // SP is 0xsssssssssssSSSS0  (4 lower bits are zero)
1082     // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
1083     //       0xSSSSPPPPPPPPPPPP
1084     SP = IRB.CreateShl(SP, 44);
1085 
1086     // Store data to ring buffer.
1087     Value *RecordPtr =
1088         IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0));
1089     IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr);
1090 
1091     // Update the ring buffer. Top byte of ThreadLong defines the size of the
1092     // buffer in pages, it must be a power of two, and the start of the buffer
1093     // must be aligned by twice that much. Therefore wrap around of the ring
1094     // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
1095     // The use of AShr instead of LShr is due to
1096     //   https://bugs.llvm.org/show_bug.cgi?id=39030
1097     // Runtime library makes sure not to use the highest bit.
1098     Value *WrapMask = IRB.CreateXor(
1099         IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
1100         ConstantInt::get(IntptrTy, (uint64_t)-1));
1101     Value *ThreadLongNew = IRB.CreateAnd(
1102         IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
1103     IRB.CreateStore(ThreadLongNew, SlotPtr);
1104   }
1105 
1106   // Get shadow base address by aligning RecordPtr up.
1107   // Note: this is not correct if the pointer is already aligned.
1108   // Runtime library will make sure this never happens.
1109   ShadowBase = IRB.CreateAdd(
1110       IRB.CreateOr(
1111           ThreadLongMaybeUntagged,
1112           ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
1113       ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
1114   ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
1115 }
1116 
1117 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
1118   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1119   Function *ReadRegister =
1120       Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
1121   MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
1122   Value *Args[] = {MetadataAsValue::get(*C, MD)};
1123   return IRB.CreateCall(ReadRegister, Args);
1124 }
1125 
1126 bool HWAddressSanitizer::instrumentLandingPads(
1127     SmallVectorImpl<Instruction *> &LandingPadVec) {
1128   for (auto *LP : LandingPadVec) {
1129     IRBuilder<> IRB(LP->getNextNode());
1130     IRB.CreateCall(
1131         HWAsanHandleVfork,
1132         {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"
1133                                                                       : "sp")});
1134   }
1135   return true;
1136 }
1137 
1138 bool HWAddressSanitizer::instrumentStack(
1139     SmallVectorImpl<AllocaInst *> &Allocas,
1140     DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap,
1141     SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) {
1142   // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1143   // alloca addresses using that. Unfortunately, offsets are not known yet
1144   // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1145   // temp, shift-OR it into each alloca address and xor with the retag mask.
1146   // This generates one extra instruction per alloca use.
1147   for (unsigned N = 0; N < Allocas.size(); ++N) {
1148     auto *AI = Allocas[N];
1149     IRBuilder<> IRB(AI->getNextNode());
1150 
1151     // Replace uses of the alloca with tagged address.
1152     Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
1153     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
1154     Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
1155     std::string Name =
1156         AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
1157     Replacement->setName(Name + ".hwasan");
1158 
1159     AI->replaceUsesWithIf(Replacement,
1160                           [AILong](Use &U) { return U.getUser() != AILong; });
1161 
1162     for (auto *DDI : AllocaDbgMap.lookup(AI)) {
1163       // Prepend "tag_offset, N" to the dwarf expression.
1164       // Tag offset logically applies to the alloca pointer, and it makes sense
1165       // to put it at the beginning of the expression.
1166       SmallVector<uint64_t, 8> NewOps = {dwarf::DW_OP_LLVM_tag_offset,
1167                                          RetagMask(N)};
1168       auto Locations = DDI->location_ops();
1169       unsigned LocNo = std::distance(Locations.begin(), find(Locations, AI));
1170       DDI->setExpression(
1171           DIExpression::appendOpsToArg(DDI->getExpression(), NewOps, LocNo));
1172     }
1173 
1174     size_t Size = getAllocaSizeInBytes(*AI);
1175     tagAlloca(IRB, AI, Tag, Size);
1176 
1177     for (auto RI : RetVec) {
1178       IRB.SetInsertPoint(RI);
1179 
1180       // Re-tag alloca memory with the special UAR tag.
1181       Value *Tag = getUARTag(IRB, StackTag);
1182       tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getObjectAlignment()));
1183     }
1184   }
1185 
1186   return true;
1187 }
1188 
1189 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
1190   return (AI.getAllocatedType()->isSized() &&
1191           // FIXME: instrument dynamic allocas, too
1192           AI.isStaticAlloca() &&
1193           // alloca() may be called with 0 size, ignore it.
1194           getAllocaSizeInBytes(AI) > 0 &&
1195           // We are only interested in allocas not promotable to registers.
1196           // Promotable allocas are common under -O0.
1197           !isAllocaPromotable(&AI) &&
1198           // inalloca allocas are not treated as static, and we don't want
1199           // dynamic alloca instrumentation for them as well.
1200           !AI.isUsedWithInAlloca() &&
1201           // swifterror allocas are register promoted by ISel
1202           !AI.isSwiftError());
1203 }
1204 
1205 bool HWAddressSanitizer::sanitizeFunction(Function &F) {
1206   if (&F == HwasanCtorFunction)
1207     return false;
1208 
1209   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
1210     return false;
1211 
1212   LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1213 
1214   SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1215   SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1216   SmallVector<AllocaInst*, 8> AllocasToInstrument;
1217   SmallVector<Instruction*, 8> RetVec;
1218   SmallVector<Instruction*, 8> LandingPadVec;
1219   DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> AllocaDbgMap;
1220   for (auto &BB : F) {
1221     for (auto &Inst : BB) {
1222       if (InstrumentStack)
1223         if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
1224           if (isInterestingAlloca(*AI))
1225             AllocasToInstrument.push_back(AI);
1226           continue;
1227         }
1228 
1229       if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
1230           isa<CleanupReturnInst>(Inst))
1231         RetVec.push_back(&Inst);
1232 
1233       if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst))
1234         for (Value *V : DVI->location_ops())
1235           if (auto *Alloca = dyn_cast_or_null<AllocaInst>(V))
1236             AllocaDbgMap[Alloca].push_back(DVI);
1237 
1238       if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
1239         LandingPadVec.push_back(&Inst);
1240 
1241       getInterestingMemoryOperands(&Inst, OperandsToInstrument);
1242 
1243       if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst))
1244         IntrinToInstrument.push_back(MI);
1245     }
1246   }
1247 
1248   initializeCallbacks(*F.getParent());
1249 
1250   bool Changed = false;
1251 
1252   if (!LandingPadVec.empty())
1253     Changed |= instrumentLandingPads(LandingPadVec);
1254 
1255   if (AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1256       F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1257     // __hwasan_personality_thunk is a no-op for functions without an
1258     // instrumented stack, so we can drop it.
1259     F.setPersonalityFn(nullptr);
1260     Changed = true;
1261   }
1262 
1263   if (AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1264       IntrinToInstrument.empty())
1265     return Changed;
1266 
1267   assert(!ShadowBase);
1268 
1269   Instruction *InsertPt = &*F.getEntryBlock().begin();
1270   IRBuilder<> EntryIRB(InsertPt);
1271   emitPrologue(EntryIRB,
1272                /*WithFrameRecord*/ ClRecordStackHistory &&
1273                    !AllocasToInstrument.empty());
1274 
1275   if (!AllocasToInstrument.empty()) {
1276     Value *StackTag =
1277         ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
1278     instrumentStack(AllocasToInstrument, AllocaDbgMap, RetVec, StackTag);
1279   }
1280   // Pad and align each of the allocas that we instrumented to stop small
1281   // uninteresting allocas from hiding in instrumented alloca's padding and so
1282   // that we have enough space to store real tags for short granules.
1283   DenseMap<AllocaInst *, AllocaInst *> AllocaToPaddedAllocaMap;
1284   for (AllocaInst *AI : AllocasToInstrument) {
1285     uint64_t Size = getAllocaSizeInBytes(*AI);
1286     uint64_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
1287     AI->setAlignment(
1288         Align(std::max(AI->getAlignment(), Mapping.getObjectAlignment())));
1289     if (Size != AlignedSize) {
1290       Type *AllocatedType = AI->getAllocatedType();
1291       if (AI->isArrayAllocation()) {
1292         uint64_t ArraySize =
1293             cast<ConstantInt>(AI->getArraySize())->getZExtValue();
1294         AllocatedType = ArrayType::get(AllocatedType, ArraySize);
1295       }
1296       Type *TypeWithPadding = StructType::get(
1297           AllocatedType, ArrayType::get(Int8Ty, AlignedSize - Size));
1298       auto *NewAI = new AllocaInst(
1299           TypeWithPadding, AI->getType()->getAddressSpace(), nullptr, "", AI);
1300       NewAI->takeName(AI);
1301       NewAI->setAlignment(AI->getAlign());
1302       NewAI->setUsedWithInAlloca(AI->isUsedWithInAlloca());
1303       NewAI->setSwiftError(AI->isSwiftError());
1304       NewAI->copyMetadata(*AI);
1305       auto *Bitcast = new BitCastInst(NewAI, AI->getType(), "", AI);
1306       AI->replaceAllUsesWith(Bitcast);
1307       AllocaToPaddedAllocaMap[AI] = NewAI;
1308     }
1309   }
1310 
1311   if (!AllocaToPaddedAllocaMap.empty()) {
1312     for (auto &BB : F) {
1313       for (auto &Inst : BB) {
1314         if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst)) {
1315           for (Value *V : DVI->location_ops()) {
1316             if (auto *AI = dyn_cast_or_null<AllocaInst>(V)) {
1317               if (auto *NewAI = AllocaToPaddedAllocaMap.lookup(AI))
1318                 DVI->replaceVariableLocationOp(V, NewAI);
1319             }
1320           }
1321         }
1322       }
1323     }
1324     for (auto &P : AllocaToPaddedAllocaMap)
1325       P.first->eraseFromParent();
1326   }
1327 
1328   // If we split the entry block, move any allocas that were originally in the
1329   // entry block back into the entry block so that they aren't treated as
1330   // dynamic allocas.
1331   if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1332     InsertPt = &*F.getEntryBlock().begin();
1333     for (auto II = EntryIRB.GetInsertBlock()->begin(),
1334               IE = EntryIRB.GetInsertBlock()->end();
1335          II != IE;) {
1336       Instruction *I = &*II++;
1337       if (auto *AI = dyn_cast<AllocaInst>(I))
1338         if (isa<ConstantInt>(AI->getArraySize()))
1339           I->moveBefore(InsertPt);
1340     }
1341   }
1342 
1343   for (auto &Operand : OperandsToInstrument)
1344     instrumentMemAccess(Operand);
1345 
1346   if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
1347     for (auto Inst : IntrinToInstrument)
1348       instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
1349   }
1350 
1351   ShadowBase = nullptr;
1352   StackBaseTag = nullptr;
1353 
1354   return true;
1355 }
1356 
1357 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1358   assert(!UsePageAliases);
1359   Constant *Initializer = GV->getInitializer();
1360   uint64_t SizeInBytes =
1361       M.getDataLayout().getTypeAllocSize(Initializer->getType());
1362   uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
1363   if (SizeInBytes != NewSize) {
1364     // Pad the initializer out to the next multiple of 16 bytes and add the
1365     // required short granule tag.
1366     std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1367     Init.back() = Tag;
1368     Constant *Padding = ConstantDataArray::get(*C, Init);
1369     Initializer = ConstantStruct::getAnon({Initializer, Padding});
1370   }
1371 
1372   auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1373                                    GlobalValue::ExternalLinkage, Initializer,
1374                                    GV->getName() + ".hwasan");
1375   NewGV->copyAttributesFrom(GV);
1376   NewGV->setLinkage(GlobalValue::PrivateLinkage);
1377   NewGV->copyMetadata(GV, 0);
1378   NewGV->setAlignment(
1379       MaybeAlign(std::max(GV->getAlignment(), Mapping.getObjectAlignment())));
1380 
1381   // It is invalid to ICF two globals that have different tags. In the case
1382   // where the size of the global is a multiple of the tag granularity the
1383   // contents of the globals may be the same but the tags (i.e. symbol values)
1384   // may be different, and the symbols are not considered during ICF. In the
1385   // case where the size is not a multiple of the granularity, the short granule
1386   // tags would discriminate two globals with different tags, but there would
1387   // otherwise be nothing stopping such a global from being incorrectly ICF'd
1388   // with an uninstrumented (i.e. tag 0) global that happened to have the short
1389   // granule tag in the last byte.
1390   NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1391 
1392   // Descriptor format (assuming little-endian):
1393   // bytes 0-3: relative address of global
1394   // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1395   // it isn't, we create multiple descriptors)
1396   // byte 7: tag
1397   auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
1398   const uint64_t MaxDescriptorSize = 0xfffff0;
1399   for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1400        DescriptorPos += MaxDescriptorSize) {
1401     auto *Descriptor =
1402         new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1403                            nullptr, GV->getName() + ".hwasan.descriptor");
1404     auto *GVRelPtr = ConstantExpr::getTrunc(
1405         ConstantExpr::getAdd(
1406             ConstantExpr::getSub(
1407                 ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1408                 ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
1409             ConstantInt::get(Int64Ty, DescriptorPos)),
1410         Int32Ty);
1411     uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
1412     auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
1413     Descriptor->setComdat(NewGV->getComdat());
1414     Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
1415     Descriptor->setSection("hwasan_globals");
1416     Descriptor->setMetadata(LLVMContext::MD_associated,
1417                             MDNode::get(*C, ValueAsMetadata::get(NewGV)));
1418     appendToCompilerUsed(M, Descriptor);
1419   }
1420 
1421   Constant *Aliasee = ConstantExpr::getIntToPtr(
1422       ConstantExpr::getAdd(
1423           ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1424           ConstantInt::get(Int64Ty, uint64_t(Tag) << kPointerTagShift)),
1425       GV->getType());
1426   auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
1427                                     GV->getLinkage(), "", Aliasee, &M);
1428   Alias->setVisibility(GV->getVisibility());
1429   Alias->takeName(GV);
1430   GV->replaceAllUsesWith(Alias);
1431   GV->eraseFromParent();
1432 }
1433 
1434 void HWAddressSanitizer::instrumentGlobals() {
1435   std::vector<GlobalVariable *> Globals;
1436   for (GlobalVariable &GV : M.globals()) {
1437     if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") ||
1438         GV.isThreadLocal())
1439       continue;
1440 
1441     // Common symbols can't have aliases point to them, so they can't be tagged.
1442     if (GV.hasCommonLinkage())
1443       continue;
1444 
1445     // Globals with custom sections may be used in __start_/__stop_ enumeration,
1446     // which would be broken both by adding tags and potentially by the extra
1447     // padding/alignment that we insert.
1448     if (GV.hasSection())
1449       continue;
1450 
1451     Globals.push_back(&GV);
1452   }
1453 
1454   MD5 Hasher;
1455   Hasher.update(M.getSourceFileName());
1456   MD5::MD5Result Hash;
1457   Hasher.final(Hash);
1458   uint8_t Tag = Hash[0];
1459 
1460   for (GlobalVariable *GV : Globals) {
1461     // Skip tag 0 in order to avoid collisions with untagged memory.
1462     if (Tag == 0)
1463       Tag = 1;
1464     instrumentGlobal(GV, Tag++);
1465   }
1466 }
1467 
1468 void HWAddressSanitizer::instrumentPersonalityFunctions() {
1469   // We need to untag stack frames as we unwind past them. That is the job of
1470   // the personality function wrapper, which either wraps an existing
1471   // personality function or acts as a personality function on its own. Each
1472   // function that has a personality function or that can be unwound past has
1473   // its personality function changed to a thunk that calls the personality
1474   // function wrapper in the runtime.
1475   MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1476   for (Function &F : M) {
1477     if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress))
1478       continue;
1479 
1480     if (F.hasPersonalityFn()) {
1481       PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F);
1482     } else if (!F.hasFnAttribute(Attribute::NoUnwind)) {
1483       PersonalityFns[nullptr].push_back(&F);
1484     }
1485   }
1486 
1487   if (PersonalityFns.empty())
1488     return;
1489 
1490   FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1491       "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty,
1492       Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy);
1493   FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy);
1494   FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy);
1495 
1496   for (auto &P : PersonalityFns) {
1497     std::string ThunkName = kHwasanPersonalityThunkName;
1498     if (P.first)
1499       ThunkName += ("." + P.first->getName()).str();
1500     FunctionType *ThunkFnTy = FunctionType::get(
1501         Int32Ty, {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int8PtrTy}, false);
1502     bool IsLocal = P.first && (!isa<GlobalValue>(P.first) ||
1503                                cast<GlobalValue>(P.first)->hasLocalLinkage());
1504     auto *ThunkFn = Function::Create(ThunkFnTy,
1505                                      IsLocal ? GlobalValue::InternalLinkage
1506                                              : GlobalValue::LinkOnceODRLinkage,
1507                                      ThunkName, &M);
1508     if (!IsLocal) {
1509       ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1510       ThunkFn->setComdat(M.getOrInsertComdat(ThunkName));
1511     }
1512 
1513     auto *BB = BasicBlock::Create(*C, "entry", ThunkFn);
1514     IRBuilder<> IRB(BB);
1515     CallInst *WrapperCall = IRB.CreateCall(
1516         HwasanPersonalityWrapper,
1517         {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2),
1518          ThunkFn->getArg(3), ThunkFn->getArg(4),
1519          P.first ? IRB.CreateBitCast(P.first, Int8PtrTy)
1520                  : Constant::getNullValue(Int8PtrTy),
1521          IRB.CreateBitCast(UnwindGetGR.getCallee(), Int8PtrTy),
1522          IRB.CreateBitCast(UnwindGetCFA.getCallee(), Int8PtrTy)});
1523     WrapperCall->setTailCall();
1524     IRB.CreateRet(WrapperCall);
1525 
1526     for (Function *F : P.second)
1527       F->setPersonalityFn(ThunkFn);
1528   }
1529 }
1530 
1531 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
1532                                              bool InstrumentWithCalls) {
1533   Scale = kDefaultShadowScale;
1534   if (TargetTriple.isOSFuchsia()) {
1535     // Fuchsia is always PIE, which means that the beginning of the address
1536     // space is always available.
1537     InGlobal = false;
1538     InTls = false;
1539     Offset = 0;
1540   } else if (ClMappingOffset.getNumOccurrences() > 0) {
1541     InGlobal = false;
1542     InTls = false;
1543     Offset = ClMappingOffset;
1544   } else if (ClEnableKhwasan || InstrumentWithCalls) {
1545     InGlobal = false;
1546     InTls = false;
1547     Offset = 0;
1548   } else if (ClWithIfunc) {
1549     InGlobal = true;
1550     InTls = false;
1551     Offset = kDynamicShadowSentinel;
1552   } else if (ClWithTls) {
1553     InGlobal = false;
1554     InTls = true;
1555     Offset = kDynamicShadowSentinel;
1556   } else {
1557     InGlobal = false;
1558     InTls = false;
1559     Offset = kDynamicShadowSentinel;
1560   }
1561 }
1562