1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file is a part of HWAddressSanitizer, an address sanity checker
11 /// based on tagged addressing.
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15 #include "llvm/ADT/MapVector.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/StringExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/BinaryFormat/ELF.h"
21 #include "llvm/IR/Attributes.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/Constant.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DebugInfoMetadata.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/IRBuilder.h"
30 #include "llvm/IR/InlineAsm.h"
31 #include "llvm/IR/InstVisitor.h"
32 #include "llvm/IR/Instruction.h"
33 #include "llvm/IR/Instructions.h"
34 #include "llvm/IR/IntrinsicInst.h"
35 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/IR/LLVMContext.h"
37 #include "llvm/IR/MDBuilder.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/IR/Value.h"
41 #include "llvm/InitializePasses.h"
42 #include "llvm/Pass.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/CommandLine.h"
45 #include "llvm/Support/Debug.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Transforms/Instrumentation.h"
48 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
49 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
50 #include "llvm/Transforms/Utils/ModuleUtils.h"
51 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
52 #include <sstream>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "hwasan"
57 
58 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
59 static const char *const kHwasanNoteName = "hwasan.note";
60 static const char *const kHwasanInitName = "__hwasan_init";
61 static const char *const kHwasanPersonalityThunkName =
62     "__hwasan_personality_thunk";
63 
64 static const char *const kHwasanShadowMemoryDynamicAddress =
65     "__hwasan_shadow_memory_dynamic_address";
66 
67 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
68 static const size_t kNumberOfAccessSizes = 5;
69 
70 static const size_t kDefaultShadowScale = 4;
71 static const uint64_t kDynamicShadowSentinel =
72     std::numeric_limits<uint64_t>::max();
73 static const unsigned kPointerTagShift = 56;
74 
75 static const unsigned kShadowBaseAlignment = 32;
76 
77 static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
78     "hwasan-memory-access-callback-prefix",
79     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
80     cl::init("__hwasan_"));
81 
82 static cl::opt<bool>
83     ClInstrumentWithCalls("hwasan-instrument-with-calls",
84                 cl::desc("instrument reads and writes with callbacks"),
85                 cl::Hidden, cl::init(false));
86 
87 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
88                                        cl::desc("instrument read instructions"),
89                                        cl::Hidden, cl::init(true));
90 
91 static cl::opt<bool> ClInstrumentWrites(
92     "hwasan-instrument-writes", cl::desc("instrument write instructions"),
93     cl::Hidden, cl::init(true));
94 
95 static cl::opt<bool> ClInstrumentAtomics(
96     "hwasan-instrument-atomics",
97     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
98     cl::init(true));
99 
100 static cl::opt<bool> ClInstrumentByval("hwasan-instrument-byval",
101                                        cl::desc("instrument byval arguments"),
102                                        cl::Hidden, cl::init(true));
103 
104 static cl::opt<bool> ClRecover(
105     "hwasan-recover",
106     cl::desc("Enable recovery mode (continue-after-error)."),
107     cl::Hidden, cl::init(false));
108 
109 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
110                                        cl::desc("instrument stack (allocas)"),
111                                        cl::Hidden, cl::init(true));
112 
113 static cl::opt<bool> ClUARRetagToZero(
114     "hwasan-uar-retag-to-zero",
115     cl::desc("Clear alloca tags before returning from the function to allow "
116              "non-instrumented and instrumented function calls mix. When set "
117              "to false, allocas are retagged before returning from the "
118              "function to detect use after return."),
119     cl::Hidden, cl::init(true));
120 
121 static cl::opt<bool> ClGenerateTagsWithCalls(
122     "hwasan-generate-tags-with-calls",
123     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
124     cl::init(false));
125 
126 static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
127                                cl::Hidden, cl::init(false), cl::ZeroOrMore);
128 
129 static cl::opt<int> ClMatchAllTag(
130     "hwasan-match-all-tag",
131     cl::desc("don't report bad accesses via pointers with this tag"),
132     cl::Hidden, cl::init(-1));
133 
134 static cl::opt<bool> ClEnableKhwasan(
135     "hwasan-kernel",
136     cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
137     cl::Hidden, cl::init(false));
138 
139 // These flags allow to change the shadow mapping and control how shadow memory
140 // is accessed. The shadow mapping looks like:
141 //    Shadow = (Mem >> scale) + offset
142 
143 static cl::opt<uint64_t>
144     ClMappingOffset("hwasan-mapping-offset",
145                     cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
146                     cl::Hidden, cl::init(0));
147 
148 static cl::opt<bool>
149     ClWithIfunc("hwasan-with-ifunc",
150                 cl::desc("Access dynamic shadow through an ifunc global on "
151                          "platforms that support this"),
152                 cl::Hidden, cl::init(false));
153 
154 static cl::opt<bool> ClWithTls(
155     "hwasan-with-tls",
156     cl::desc("Access dynamic shadow through an thread-local pointer on "
157              "platforms that support this"),
158     cl::Hidden, cl::init(true));
159 
160 static cl::opt<bool>
161     ClRecordStackHistory("hwasan-record-stack-history",
162                          cl::desc("Record stack frames with tagged allocations "
163                                   "in a thread-local ring buffer"),
164                          cl::Hidden, cl::init(true));
165 static cl::opt<bool>
166     ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
167                               cl::desc("instrument memory intrinsics"),
168                               cl::Hidden, cl::init(true));
169 
170 static cl::opt<bool>
171     ClInstrumentLandingPads("hwasan-instrument-landing-pads",
172                             cl::desc("instrument landing pads"), cl::Hidden,
173                             cl::init(false), cl::ZeroOrMore);
174 
175 static cl::opt<bool> ClUseShortGranules(
176     "hwasan-use-short-granules",
177     cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
178     cl::init(false), cl::ZeroOrMore);
179 
180 static cl::opt<bool> ClInstrumentPersonalityFunctions(
181     "hwasan-instrument-personality-functions",
182     cl::desc("instrument personality functions"), cl::Hidden, cl::init(false),
183     cl::ZeroOrMore);
184 
185 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
186                                        cl::desc("inline all checks"),
187                                        cl::Hidden, cl::init(false));
188 
189 namespace {
190 
191 /// An instrumentation pass implementing detection of addressability bugs
192 /// using tagged pointers.
193 class HWAddressSanitizer {
194 public:
195   explicit HWAddressSanitizer(Module &M, bool CompileKernel = false,
196                               bool Recover = false) : M(M) {
197     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
198     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
199         ClEnableKhwasan : CompileKernel;
200 
201     initializeModule();
202   }
203 
204   bool sanitizeFunction(Function &F);
205   void initializeModule();
206   void createHwasanCtorComdat();
207 
208   void initializeCallbacks(Module &M);
209 
210   Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
211 
212   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
213   Value *getShadowNonTls(IRBuilder<> &IRB);
214 
215   void untagPointerOperand(Instruction *I, Value *Addr);
216   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
217   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
218                                  unsigned AccessSizeIndex,
219                                  Instruction *InsertBefore);
220   void instrumentMemIntrinsic(MemIntrinsic *MI);
221   bool instrumentMemAccess(InterestingMemoryOperand &O);
222   bool ignoreAccess(Value *Ptr);
223   void getInterestingMemoryOperands(
224       Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
225 
226   bool isInterestingAlloca(const AllocaInst &AI);
227   bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
228   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
229   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
230   bool instrumentStack(
231       SmallVectorImpl<AllocaInst *> &Allocas,
232       DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap,
233       SmallVectorImpl<Instruction *> &RetVec, Value *StackTag);
234   Value *readRegister(IRBuilder<> &IRB, StringRef Name);
235   bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
236   Value *getNextTagWithCall(IRBuilder<> &IRB);
237   Value *getStackBaseTag(IRBuilder<> &IRB);
238   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
239                      unsigned AllocaNo);
240   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
241 
242   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
243   void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
244 
245   void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
246   void instrumentGlobals();
247 
248   void instrumentPersonalityFunctions();
249 
250 private:
251   LLVMContext *C;
252   Module &M;
253   Triple TargetTriple;
254   FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
255   FunctionCallee HWAsanHandleVfork;
256 
257   /// This struct defines the shadow mapping using the rule:
258   ///   shadow = (mem >> Scale) + Offset.
259   /// If InGlobal is true, then
260   ///   extern char __hwasan_shadow[];
261   ///   shadow = (mem >> Scale) + &__hwasan_shadow
262   /// If InTls is true, then
263   ///   extern char *__hwasan_tls;
264   ///   shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
265   struct ShadowMapping {
266     int Scale;
267     uint64_t Offset;
268     bool InGlobal;
269     bool InTls;
270 
271     void init(Triple &TargetTriple);
272     unsigned getObjectAlignment() const { return 1U << Scale; }
273   };
274   ShadowMapping Mapping;
275 
276   Type *VoidTy = Type::getVoidTy(M.getContext());
277   Type *IntptrTy;
278   Type *Int8PtrTy;
279   Type *Int8Ty;
280   Type *Int32Ty;
281   Type *Int64Ty = Type::getInt64Ty(M.getContext());
282 
283   bool CompileKernel;
284   bool Recover;
285   bool OutlinedChecks;
286   bool UseShortGranules;
287   bool InstrumentLandingPads;
288 
289   bool HasMatchAllTag = false;
290   uint8_t MatchAllTag = 0;
291 
292   Function *HwasanCtorFunction;
293 
294   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
295   FunctionCallee HwasanMemoryAccessCallbackSized[2];
296 
297   FunctionCallee HwasanTagMemoryFunc;
298   FunctionCallee HwasanGenerateTagFunc;
299 
300   Constant *ShadowGlobal;
301 
302   Value *ShadowBase = nullptr;
303   Value *StackBaseTag = nullptr;
304   GlobalValue *ThreadPtrGlobal = nullptr;
305 };
306 
307 class HWAddressSanitizerLegacyPass : public FunctionPass {
308 public:
309   // Pass identification, replacement for typeid.
310   static char ID;
311 
312   explicit HWAddressSanitizerLegacyPass(bool CompileKernel = false,
313                                         bool Recover = false)
314       : FunctionPass(ID), CompileKernel(CompileKernel), Recover(Recover) {
315     initializeHWAddressSanitizerLegacyPassPass(
316         *PassRegistry::getPassRegistry());
317   }
318 
319   StringRef getPassName() const override { return "HWAddressSanitizer"; }
320 
321   bool doInitialization(Module &M) override {
322     HWASan = std::make_unique<HWAddressSanitizer>(M, CompileKernel, Recover);
323     return true;
324   }
325 
326   bool runOnFunction(Function &F) override {
327     return HWASan->sanitizeFunction(F);
328   }
329 
330   bool doFinalization(Module &M) override {
331     HWASan.reset();
332     return false;
333   }
334 
335 private:
336   std::unique_ptr<HWAddressSanitizer> HWASan;
337   bool CompileKernel;
338   bool Recover;
339 };
340 
341 } // end anonymous namespace
342 
343 char HWAddressSanitizerLegacyPass::ID = 0;
344 
345 INITIALIZE_PASS_BEGIN(
346     HWAddressSanitizerLegacyPass, "hwasan",
347     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
348     false)
349 INITIALIZE_PASS_END(
350     HWAddressSanitizerLegacyPass, "hwasan",
351     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
352     false)
353 
354 FunctionPass *llvm::createHWAddressSanitizerLegacyPassPass(bool CompileKernel,
355                                                            bool Recover) {
356   assert(!CompileKernel || Recover);
357   return new HWAddressSanitizerLegacyPass(CompileKernel, Recover);
358 }
359 
360 HWAddressSanitizerPass::HWAddressSanitizerPass(bool CompileKernel, bool Recover)
361     : CompileKernel(CompileKernel), Recover(Recover) {}
362 
363 PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
364                                               ModuleAnalysisManager &MAM) {
365   HWAddressSanitizer HWASan(M, CompileKernel, Recover);
366   bool Modified = false;
367   for (Function &F : M)
368     Modified |= HWASan.sanitizeFunction(F);
369   if (Modified)
370     return PreservedAnalyses::none();
371   return PreservedAnalyses::all();
372 }
373 
374 void HWAddressSanitizer::createHwasanCtorComdat() {
375   std::tie(HwasanCtorFunction, std::ignore) =
376       getOrCreateSanitizerCtorAndInitFunctions(
377           M, kHwasanModuleCtorName, kHwasanInitName,
378           /*InitArgTypes=*/{},
379           /*InitArgs=*/{},
380           // This callback is invoked when the functions are created the first
381           // time. Hook them into the global ctors list in that case:
382           [&](Function *Ctor, FunctionCallee) {
383             Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
384             Ctor->setComdat(CtorComdat);
385             appendToGlobalCtors(M, Ctor, 0, Ctor);
386           });
387 
388   // Create a note that contains pointers to the list of global
389   // descriptors. Adding a note to the output file will cause the linker to
390   // create a PT_NOTE program header pointing to the note that we can use to
391   // find the descriptor list starting from the program headers. A function
392   // provided by the runtime initializes the shadow memory for the globals by
393   // accessing the descriptor list via the note. The dynamic loader needs to
394   // call this function whenever a library is loaded.
395   //
396   // The reason why we use a note for this instead of a more conventional
397   // approach of having a global constructor pass a descriptor list pointer to
398   // the runtime is because of an order of initialization problem. With
399   // constructors we can encounter the following problematic scenario:
400   //
401   // 1) library A depends on library B and also interposes one of B's symbols
402   // 2) B's constructors are called before A's (as required for correctness)
403   // 3) during construction, B accesses one of its "own" globals (actually
404   //    interposed by A) and triggers a HWASAN failure due to the initialization
405   //    for A not having happened yet
406   //
407   // Even without interposition it is possible to run into similar situations in
408   // cases where two libraries mutually depend on each other.
409   //
410   // We only need one note per binary, so put everything for the note in a
411   // comdat. This needs to be a comdat with an .init_array section to prevent
412   // newer versions of lld from discarding the note.
413   //
414   // Create the note even if we aren't instrumenting globals. This ensures that
415   // binaries linked from object files with both instrumented and
416   // non-instrumented globals will end up with a note, even if a comdat from an
417   // object file with non-instrumented globals is selected. The note is harmless
418   // if the runtime doesn't support it, since it will just be ignored.
419   Comdat *NoteComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
420 
421   Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
422   auto Start =
423       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
424                          nullptr, "__start_hwasan_globals");
425   Start->setVisibility(GlobalValue::HiddenVisibility);
426   Start->setDSOLocal(true);
427   auto Stop =
428       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
429                          nullptr, "__stop_hwasan_globals");
430   Stop->setVisibility(GlobalValue::HiddenVisibility);
431   Stop->setDSOLocal(true);
432 
433   // Null-terminated so actually 8 bytes, which are required in order to align
434   // the note properly.
435   auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
436 
437   auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
438                                  Int32Ty, Int32Ty);
439   auto *Note =
440       new GlobalVariable(M, NoteTy, /*isConstant=*/true,
441                          GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
442   Note->setSection(".note.hwasan.globals");
443   Note->setComdat(NoteComdat);
444   Note->setAlignment(Align(4));
445   Note->setDSOLocal(true);
446 
447   // The pointers in the note need to be relative so that the note ends up being
448   // placed in rodata, which is the standard location for notes.
449   auto CreateRelPtr = [&](Constant *Ptr) {
450     return ConstantExpr::getTrunc(
451         ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
452                              ConstantExpr::getPtrToInt(Note, Int64Ty)),
453         Int32Ty);
454   };
455   Note->setInitializer(ConstantStruct::getAnon(
456       {ConstantInt::get(Int32Ty, 8),                           // n_namesz
457        ConstantInt::get(Int32Ty, 8),                           // n_descsz
458        ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
459        Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
460   appendToCompilerUsed(M, Note);
461 
462   // Create a zero-length global in hwasan_globals so that the linker will
463   // always create start and stop symbols.
464   auto Dummy = new GlobalVariable(
465       M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
466       Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
467   Dummy->setSection("hwasan_globals");
468   Dummy->setComdat(NoteComdat);
469   Dummy->setMetadata(LLVMContext::MD_associated,
470                      MDNode::get(*C, ValueAsMetadata::get(Note)));
471   appendToCompilerUsed(M, Dummy);
472 }
473 
474 /// Module-level initialization.
475 ///
476 /// inserts a call to __hwasan_init to the module's constructor list.
477 void HWAddressSanitizer::initializeModule() {
478   LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
479   auto &DL = M.getDataLayout();
480 
481   TargetTriple = Triple(M.getTargetTriple());
482 
483   Mapping.init(TargetTriple);
484 
485   C = &(M.getContext());
486   IRBuilder<> IRB(*C);
487   IntptrTy = IRB.getIntPtrTy(DL);
488   Int8PtrTy = IRB.getInt8PtrTy();
489   Int8Ty = IRB.getInt8Ty();
490   Int32Ty = IRB.getInt32Ty();
491 
492   HwasanCtorFunction = nullptr;
493 
494   // Older versions of Android do not have the required runtime support for
495   // short granules, global or personality function instrumentation. On other
496   // platforms we currently require using the latest version of the runtime.
497   bool NewRuntime =
498       !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
499 
500   UseShortGranules =
501       ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
502   OutlinedChecks =
503       TargetTriple.isAArch64() && TargetTriple.isOSBinFormatELF() &&
504       (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
505 
506   if (ClMatchAllTag.getNumOccurrences()) {
507     if (ClMatchAllTag != -1) {
508       HasMatchAllTag = true;
509       MatchAllTag = ClMatchAllTag & 0xFF;
510     }
511   } else if (CompileKernel) {
512     HasMatchAllTag = true;
513     MatchAllTag = 0xFF;
514   }
515 
516   // If we don't have personality function support, fall back to landing pads.
517   InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
518                               ? ClInstrumentLandingPads
519                               : !NewRuntime;
520 
521   if (!CompileKernel) {
522     createHwasanCtorComdat();
523     bool InstrumentGlobals =
524         ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
525     if (InstrumentGlobals)
526       instrumentGlobals();
527 
528     bool InstrumentPersonalityFunctions =
529         ClInstrumentPersonalityFunctions.getNumOccurrences()
530             ? ClInstrumentPersonalityFunctions
531             : NewRuntime;
532     if (InstrumentPersonalityFunctions)
533       instrumentPersonalityFunctions();
534   }
535 
536   if (!TargetTriple.isAndroid()) {
537     Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
538       auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
539                                     GlobalValue::ExternalLinkage, nullptr,
540                                     "__hwasan_tls", nullptr,
541                                     GlobalVariable::InitialExecTLSModel);
542       appendToCompilerUsed(M, GV);
543       return GV;
544     });
545     ThreadPtrGlobal = cast<GlobalVariable>(C);
546   }
547 }
548 
549 void HWAddressSanitizer::initializeCallbacks(Module &M) {
550   IRBuilder<> IRB(*C);
551   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
552     const std::string TypeStr = AccessIsWrite ? "store" : "load";
553     const std::string EndingStr = Recover ? "_noabort" : "";
554 
555     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
556         ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
557         FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
558 
559     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
560          AccessSizeIndex++) {
561       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
562           M.getOrInsertFunction(
563               ClMemoryAccessCallbackPrefix + TypeStr +
564                   itostr(1ULL << AccessSizeIndex) + EndingStr,
565               FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
566     }
567   }
568 
569   HwasanTagMemoryFunc = M.getOrInsertFunction(
570       "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
571   HwasanGenerateTagFunc =
572       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
573 
574   ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
575                                      ArrayType::get(IRB.getInt8Ty(), 0));
576 
577   const std::string MemIntrinCallbackPrefix =
578       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
579   HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
580                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
581                                         IRB.getInt8PtrTy(), IntptrTy);
582   HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
583                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
584                                        IRB.getInt8PtrTy(), IntptrTy);
585   HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
586                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
587                                        IRB.getInt32Ty(), IntptrTy);
588 
589   HWAsanHandleVfork =
590       M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy);
591 }
592 
593 Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
594   // An empty inline asm with input reg == output reg.
595   // An opaque no-op cast, basically.
596   // This prevents code bloat as a result of rematerializing trivial definitions
597   // such as constants or global addresses at every load and store.
598   InlineAsm *Asm =
599       InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
600                      StringRef(""), StringRef("=r,0"),
601                      /*hasSideEffects=*/false);
602   return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
603 }
604 
605 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
606   return getOpaqueNoopCast(IRB, ShadowGlobal);
607 }
608 
609 Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
610   if (Mapping.Offset != kDynamicShadowSentinel)
611     return getOpaqueNoopCast(
612         IRB, ConstantExpr::getIntToPtr(
613                  ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy));
614 
615   if (Mapping.InGlobal) {
616     return getDynamicShadowIfunc(IRB);
617   } else {
618     Value *GlobalDynamicAddress =
619         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
620             kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
621     return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
622   }
623 }
624 
625 bool HWAddressSanitizer::ignoreAccess(Value *Ptr) {
626   // Do not instrument acesses from different address spaces; we cannot deal
627   // with them.
628   Type *PtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
629   if (PtrTy->getPointerAddressSpace() != 0)
630     return true;
631 
632   // Ignore swifterror addresses.
633   // swifterror memory addresses are mem2reg promoted by instruction
634   // selection. As such they cannot have regular uses like an instrumentation
635   // function and it makes no sense to track them as memory.
636   if (Ptr->isSwiftError())
637     return true;
638 
639   return false;
640 }
641 
642 void HWAddressSanitizer::getInterestingMemoryOperands(
643     Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
644   // Skip memory accesses inserted by another instrumentation.
645   if (I->hasMetadata("nosanitize"))
646     return;
647 
648   // Do not instrument the load fetching the dynamic shadow address.
649   if (ShadowBase == I)
650     return;
651 
652   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
653     if (!ClInstrumentReads || ignoreAccess(LI->getPointerOperand()))
654       return;
655     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
656                              LI->getType(), LI->getAlign());
657   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
658     if (!ClInstrumentWrites || ignoreAccess(SI->getPointerOperand()))
659       return;
660     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
661                              SI->getValueOperand()->getType(), SI->getAlign());
662   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
663     if (!ClInstrumentAtomics || ignoreAccess(RMW->getPointerOperand()))
664       return;
665     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
666                              RMW->getValOperand()->getType(), None);
667   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
668     if (!ClInstrumentAtomics || ignoreAccess(XCHG->getPointerOperand()))
669       return;
670     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
671                              XCHG->getCompareOperand()->getType(), None);
672   } else if (auto CI = dyn_cast<CallInst>(I)) {
673     for (unsigned ArgNo = 0; ArgNo < CI->getNumArgOperands(); ArgNo++) {
674       if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
675           ignoreAccess(CI->getArgOperand(ArgNo)))
676         continue;
677       Type *Ty = CI->getParamByValType(ArgNo);
678       Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
679     }
680   }
681 }
682 
683 static unsigned getPointerOperandIndex(Instruction *I) {
684   if (LoadInst *LI = dyn_cast<LoadInst>(I))
685     return LI->getPointerOperandIndex();
686   if (StoreInst *SI = dyn_cast<StoreInst>(I))
687     return SI->getPointerOperandIndex();
688   if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
689     return RMW->getPointerOperandIndex();
690   if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
691     return XCHG->getPointerOperandIndex();
692   report_fatal_error("Unexpected instruction");
693   return -1;
694 }
695 
696 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
697   size_t Res = countTrailingZeros(TypeSize / 8);
698   assert(Res < kNumberOfAccessSizes);
699   return Res;
700 }
701 
702 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
703   if (TargetTriple.isAArch64())
704     return;
705 
706   IRBuilder<> IRB(I);
707   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
708   Value *UntaggedPtr =
709       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
710   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
711 }
712 
713 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
714   // Mem >> Scale
715   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
716   if (Mapping.Offset == 0)
717     return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
718   // (Mem >> Scale) + Offset
719   return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow);
720 }
721 
722 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
723                                                    unsigned AccessSizeIndex,
724                                                    Instruction *InsertBefore) {
725   const int64_t AccessInfo =
726       (CompileKernel << HWASanAccessInfo::CompileKernelShift) +
727       (HasMatchAllTag << HWASanAccessInfo::HasMatchAllShift) +
728       (MatchAllTag << HWASanAccessInfo::MatchAllShift) +
729       (Recover << HWASanAccessInfo::RecoverShift) +
730       (IsWrite << HWASanAccessInfo::IsWriteShift) +
731       (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
732   IRBuilder<> IRB(InsertBefore);
733 
734   if (OutlinedChecks) {
735     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
736     Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
737     IRB.CreateCall(Intrinsic::getDeclaration(
738                        M, UseShortGranules
739                               ? Intrinsic::hwasan_check_memaccess_shortgranules
740                               : Intrinsic::hwasan_check_memaccess),
741                    {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
742     return;
743   }
744 
745   Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
746   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
747                                   IRB.getInt8Ty());
748   Value *AddrLong = untagPointer(IRB, PtrLong);
749   Value *Shadow = memToShadow(AddrLong, IRB);
750   Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
751   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
752 
753   if (HasMatchAllTag) {
754     Value *TagNotIgnored = IRB.CreateICmpNE(
755         PtrTag, ConstantInt::get(PtrTag->getType(), MatchAllTag));
756     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
757   }
758 
759   Instruction *CheckTerm =
760       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
761                                 MDBuilder(*C).createBranchWeights(1, 100000));
762 
763   IRB.SetInsertPoint(CheckTerm);
764   Value *OutOfShortGranuleTagRange =
765       IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15));
766   Instruction *CheckFailTerm =
767       SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover,
768                                 MDBuilder(*C).createBranchWeights(1, 100000));
769 
770   IRB.SetInsertPoint(CheckTerm);
771   Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty);
772   PtrLowBits = IRB.CreateAdd(
773       PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1));
774   Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag);
775   SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false,
776                             MDBuilder(*C).createBranchWeights(1, 100000),
777                             nullptr, nullptr, CheckFailTerm->getParent());
778 
779   IRB.SetInsertPoint(CheckTerm);
780   Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15);
781   InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy);
782   Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);
783   Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag);
784   SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false,
785                             MDBuilder(*C).createBranchWeights(1, 100000),
786                             nullptr, nullptr, CheckFailTerm->getParent());
787 
788   IRB.SetInsertPoint(CheckFailTerm);
789   InlineAsm *Asm;
790   switch (TargetTriple.getArch()) {
791     case Triple::x86_64:
792       // The signal handler will find the data address in rdi.
793       Asm = InlineAsm::get(
794           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
795           "int3\nnopl " +
796               itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
797               "(%rax)",
798           "{rdi}",
799           /*hasSideEffects=*/true);
800       break;
801     case Triple::aarch64:
802     case Triple::aarch64_be:
803       // The signal handler will find the data address in x0.
804       Asm = InlineAsm::get(
805           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
806           "brk #" +
807               itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
808           "{x0}",
809           /*hasSideEffects=*/true);
810       break;
811     default:
812       report_fatal_error("unsupported architecture");
813   }
814   IRB.CreateCall(Asm, PtrLong);
815   if (Recover)
816     cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent());
817 }
818 
819 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
820   IRBuilder<> IRB(MI);
821   if (isa<MemTransferInst>(MI)) {
822     IRB.CreateCall(
823         isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
824         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
825          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
826          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
827   } else if (isa<MemSetInst>(MI)) {
828     IRB.CreateCall(
829         HWAsanMemset,
830         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
831          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
832          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
833   }
834   MI->eraseFromParent();
835 }
836 
837 bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
838   Value *Addr = O.getPtr();
839 
840   LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
841 
842   if (O.MaybeMask)
843     return false; //FIXME
844 
845   IRBuilder<> IRB(O.getInsn());
846   if (isPowerOf2_64(O.TypeSize) &&
847       (O.TypeSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
848       (!O.Alignment || *O.Alignment >= (1ULL << Mapping.Scale) ||
849        *O.Alignment >= O.TypeSize / 8)) {
850     size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeSize);
851     if (ClInstrumentWithCalls) {
852       IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
853                      IRB.CreatePointerCast(Addr, IntptrTy));
854     } else {
855       instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
856     }
857   } else {
858     IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite],
859                    {IRB.CreatePointerCast(Addr, IntptrTy),
860                     ConstantInt::get(IntptrTy, O.TypeSize / 8)});
861   }
862   untagPointerOperand(O.getInsn(), Addr);
863 
864   return true;
865 }
866 
867 static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
868   uint64_t ArraySize = 1;
869   if (AI.isArrayAllocation()) {
870     const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize());
871     assert(CI && "non-constant array size");
872     ArraySize = CI->getZExtValue();
873   }
874   Type *Ty = AI.getAllocatedType();
875   uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
876   return SizeInBytes * ArraySize;
877 }
878 
879 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
880                                    Value *Tag, size_t Size) {
881   size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
882   if (!UseShortGranules)
883     Size = AlignedSize;
884 
885   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
886   if (ClInstrumentWithCalls) {
887     IRB.CreateCall(HwasanTagMemoryFunc,
888                    {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
889                     ConstantInt::get(IntptrTy, AlignedSize)});
890   } else {
891     size_t ShadowSize = Size >> Mapping.Scale;
892     Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
893     // If this memset is not inlined, it will be intercepted in the hwasan
894     // runtime library. That's OK, because the interceptor skips the checks if
895     // the address is in the shadow region.
896     // FIXME: the interceptor is not as fast as real memset. Consider lowering
897     // llvm.memset right here into either a sequence of stores, or a call to
898     // hwasan_tag_memory.
899     if (ShadowSize)
900       IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, Align(1));
901     if (Size != AlignedSize) {
902       IRB.CreateStore(
903           ConstantInt::get(Int8Ty, Size % Mapping.getObjectAlignment()),
904           IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
905       IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32(
906                                    Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy),
907                                    AlignedSize - 1));
908     }
909   }
910   return true;
911 }
912 
913 static unsigned RetagMask(unsigned AllocaNo) {
914   // A list of 8-bit numbers that have at most one run of non-zero bits.
915   // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
916   // masks.
917   // The list does not include the value 255, which is used for UAR.
918   //
919   // Because we are more likely to use earlier elements of this list than later
920   // ones, it is sorted in increasing order of probability of collision with a
921   // mask allocated (temporally) nearby. The program that generated this list
922   // can be found at:
923   // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
924   static unsigned FastMasks[] = {0,  128, 64,  192, 32,  96,  224, 112, 240,
925                                  48, 16,  120, 248, 56,  24,  8,   124, 252,
926                                  60, 28,  12,  4,   126, 254, 62,  30,  14,
927                                  6,  2,   127, 63,  31,  15,  7,   3,   1};
928   return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))];
929 }
930 
931 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
932   return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
933 }
934 
935 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
936   if (ClGenerateTagsWithCalls)
937     return getNextTagWithCall(IRB);
938   if (StackBaseTag)
939     return StackBaseTag;
940   // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
941   // first).
942   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
943   auto GetStackPointerFn = Intrinsic::getDeclaration(
944       M, Intrinsic::frameaddress,
945       IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
946   Value *StackPointer = IRB.CreateCall(
947       GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())});
948 
949   // Extract some entropy from the stack pointer for the tags.
950   // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
951   // between functions).
952   Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy);
953   Value *StackTag =
954       IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20),
955                     "hwasan.stack.base.tag");
956   return StackTag;
957 }
958 
959 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
960                                         AllocaInst *AI, unsigned AllocaNo) {
961   if (ClGenerateTagsWithCalls)
962     return getNextTagWithCall(IRB);
963   return IRB.CreateXor(StackTag,
964                        ConstantInt::get(IntptrTy, RetagMask(AllocaNo)));
965 }
966 
967 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
968   if (ClUARRetagToZero)
969     return ConstantInt::get(IntptrTy, 0);
970   if (ClGenerateTagsWithCalls)
971     return getNextTagWithCall(IRB);
972   return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
973 }
974 
975 // Add a tag to an address.
976 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
977                                       Value *PtrLong, Value *Tag) {
978   Value *TaggedPtrLong;
979   if (CompileKernel) {
980     // Kernel addresses have 0xFF in the most significant byte.
981     Value *ShiftedTag = IRB.CreateOr(
982         IRB.CreateShl(Tag, kPointerTagShift),
983         ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
984     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
985   } else {
986     // Userspace can simply do OR (tag << 56);
987     Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
988     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
989   }
990   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
991 }
992 
993 // Remove tag from an address.
994 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
995   Value *UntaggedPtrLong;
996   if (CompileKernel) {
997     // Kernel addresses have 0xFF in the most significant byte.
998     UntaggedPtrLong = IRB.CreateOr(PtrLong,
999         ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
1000   } else {
1001     // Userspace addresses have 0x00.
1002     UntaggedPtrLong = IRB.CreateAnd(PtrLong,
1003         ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
1004   }
1005   return UntaggedPtrLong;
1006 }
1007 
1008 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
1009   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1010   if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
1011     // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1012     // in Bionic's libc/private/bionic_tls.h.
1013     Function *ThreadPointerFunc =
1014         Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
1015     Value *SlotPtr = IRB.CreatePointerCast(
1016         IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
1017                                IRB.CreateCall(ThreadPointerFunc), 0x30),
1018         Ty->getPointerTo(0));
1019     return SlotPtr;
1020   }
1021   if (ThreadPtrGlobal)
1022     return ThreadPtrGlobal;
1023 
1024 
1025   return nullptr;
1026 }
1027 
1028 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1029   if (!Mapping.InTls) {
1030     ShadowBase = getShadowNonTls(IRB);
1031     return;
1032   }
1033 
1034   if (!WithFrameRecord && TargetTriple.isAndroid()) {
1035     ShadowBase = getDynamicShadowIfunc(IRB);
1036     return;
1037   }
1038 
1039   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
1040   assert(SlotPtr);
1041 
1042   Value *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
1043   // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
1044   Value *ThreadLongMaybeUntagged =
1045       TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong);
1046 
1047   if (WithFrameRecord) {
1048     Function *F = IRB.GetInsertBlock()->getParent();
1049     StackBaseTag = IRB.CreateAShr(ThreadLong, 3);
1050 
1051     // Prepare ring buffer data.
1052     Value *PC;
1053     if (TargetTriple.getArch() == Triple::aarch64)
1054       PC = readRegister(IRB, "pc");
1055     else
1056       PC = IRB.CreatePtrToInt(F, IntptrTy);
1057     Module *M = F->getParent();
1058     auto GetStackPointerFn = Intrinsic::getDeclaration(
1059         M, Intrinsic::frameaddress,
1060         IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
1061     Value *SP = IRB.CreatePtrToInt(
1062         IRB.CreateCall(GetStackPointerFn,
1063                        {Constant::getNullValue(IRB.getInt32Ty())}),
1064         IntptrTy);
1065     // Mix SP and PC.
1066     // Assumptions:
1067     // PC is 0x0000PPPPPPPPPPPP  (48 bits are meaningful, others are zero)
1068     // SP is 0xsssssssssssSSSS0  (4 lower bits are zero)
1069     // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
1070     //       0xSSSSPPPPPPPPPPPP
1071     SP = IRB.CreateShl(SP, 44);
1072 
1073     // Store data to ring buffer.
1074     Value *RecordPtr =
1075         IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0));
1076     IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr);
1077 
1078     // Update the ring buffer. Top byte of ThreadLong defines the size of the
1079     // buffer in pages, it must be a power of two, and the start of the buffer
1080     // must be aligned by twice that much. Therefore wrap around of the ring
1081     // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
1082     // The use of AShr instead of LShr is due to
1083     //   https://bugs.llvm.org/show_bug.cgi?id=39030
1084     // Runtime library makes sure not to use the highest bit.
1085     Value *WrapMask = IRB.CreateXor(
1086         IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
1087         ConstantInt::get(IntptrTy, (uint64_t)-1));
1088     Value *ThreadLongNew = IRB.CreateAnd(
1089         IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
1090     IRB.CreateStore(ThreadLongNew, SlotPtr);
1091   }
1092 
1093   // Get shadow base address by aligning RecordPtr up.
1094   // Note: this is not correct if the pointer is already aligned.
1095   // Runtime library will make sure this never happens.
1096   ShadowBase = IRB.CreateAdd(
1097       IRB.CreateOr(
1098           ThreadLongMaybeUntagged,
1099           ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
1100       ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
1101   ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
1102 }
1103 
1104 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
1105   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1106   Function *ReadRegister =
1107       Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
1108   MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
1109   Value *Args[] = {MetadataAsValue::get(*C, MD)};
1110   return IRB.CreateCall(ReadRegister, Args);
1111 }
1112 
1113 bool HWAddressSanitizer::instrumentLandingPads(
1114     SmallVectorImpl<Instruction *> &LandingPadVec) {
1115   for (auto *LP : LandingPadVec) {
1116     IRBuilder<> IRB(LP->getNextNode());
1117     IRB.CreateCall(
1118         HWAsanHandleVfork,
1119         {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"
1120                                                                       : "sp")});
1121   }
1122   return true;
1123 }
1124 
1125 bool HWAddressSanitizer::instrumentStack(
1126     SmallVectorImpl<AllocaInst *> &Allocas,
1127     DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap,
1128     SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) {
1129   // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1130   // alloca addresses using that. Unfortunately, offsets are not known yet
1131   // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1132   // temp, shift-OR it into each alloca address and xor with the retag mask.
1133   // This generates one extra instruction per alloca use.
1134   for (unsigned N = 0; N < Allocas.size(); ++N) {
1135     auto *AI = Allocas[N];
1136     IRBuilder<> IRB(AI->getNextNode());
1137 
1138     // Replace uses of the alloca with tagged address.
1139     Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
1140     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
1141     Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
1142     std::string Name =
1143         AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
1144     Replacement->setName(Name + ".hwasan");
1145 
1146     AI->replaceUsesWithIf(Replacement,
1147                           [AILong](Use &U) { return U.getUser() != AILong; });
1148 
1149     for (auto *DDI : AllocaDbgMap.lookup(AI)) {
1150       // Prepend "tag_offset, N" to the dwarf expression.
1151       // Tag offset logically applies to the alloca pointer, and it makes sense
1152       // to put it at the beginning of the expression.
1153       SmallVector<uint64_t, 8> NewOps = {dwarf::DW_OP_LLVM_tag_offset,
1154                                          RetagMask(N)};
1155       DDI->setArgOperand(
1156           2, MetadataAsValue::get(*C, DIExpression::prependOpcodes(
1157                                           DDI->getExpression(), NewOps)));
1158     }
1159 
1160     size_t Size = getAllocaSizeInBytes(*AI);
1161     tagAlloca(IRB, AI, Tag, Size);
1162 
1163     for (auto RI : RetVec) {
1164       IRB.SetInsertPoint(RI);
1165 
1166       // Re-tag alloca memory with the special UAR tag.
1167       Value *Tag = getUARTag(IRB, StackTag);
1168       tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getObjectAlignment()));
1169     }
1170   }
1171 
1172   return true;
1173 }
1174 
1175 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
1176   return (AI.getAllocatedType()->isSized() &&
1177           // FIXME: instrument dynamic allocas, too
1178           AI.isStaticAlloca() &&
1179           // alloca() may be called with 0 size, ignore it.
1180           getAllocaSizeInBytes(AI) > 0 &&
1181           // We are only interested in allocas not promotable to registers.
1182           // Promotable allocas are common under -O0.
1183           !isAllocaPromotable(&AI) &&
1184           // inalloca allocas are not treated as static, and we don't want
1185           // dynamic alloca instrumentation for them as well.
1186           !AI.isUsedWithInAlloca() &&
1187           // swifterror allocas are register promoted by ISel
1188           !AI.isSwiftError());
1189 }
1190 
1191 bool HWAddressSanitizer::sanitizeFunction(Function &F) {
1192   if (&F == HwasanCtorFunction)
1193     return false;
1194 
1195   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
1196     return false;
1197 
1198   LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1199 
1200   SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1201   SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1202   SmallVector<AllocaInst*, 8> AllocasToInstrument;
1203   SmallVector<Instruction*, 8> RetVec;
1204   SmallVector<Instruction*, 8> LandingPadVec;
1205   DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> AllocaDbgMap;
1206   for (auto &BB : F) {
1207     for (auto &Inst : BB) {
1208       if (ClInstrumentStack)
1209         if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
1210           if (isInterestingAlloca(*AI))
1211             AllocasToInstrument.push_back(AI);
1212           continue;
1213         }
1214 
1215       if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
1216           isa<CleanupReturnInst>(Inst))
1217         RetVec.push_back(&Inst);
1218 
1219       if (auto *DDI = dyn_cast<DbgVariableIntrinsic>(&Inst))
1220         if (auto *Alloca =
1221                 dyn_cast_or_null<AllocaInst>(DDI->getVariableLocation()))
1222           AllocaDbgMap[Alloca].push_back(DDI);
1223 
1224       if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
1225         LandingPadVec.push_back(&Inst);
1226 
1227       getInterestingMemoryOperands(&Inst, OperandsToInstrument);
1228 
1229       if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst))
1230         IntrinToInstrument.push_back(MI);
1231     }
1232   }
1233 
1234   initializeCallbacks(*F.getParent());
1235 
1236   bool Changed = false;
1237 
1238   if (!LandingPadVec.empty())
1239     Changed |= instrumentLandingPads(LandingPadVec);
1240 
1241   if (AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1242       F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1243     // __hwasan_personality_thunk is a no-op for functions without an
1244     // instrumented stack, so we can drop it.
1245     F.setPersonalityFn(nullptr);
1246     Changed = true;
1247   }
1248 
1249   if (AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1250       IntrinToInstrument.empty())
1251     return Changed;
1252 
1253   assert(!ShadowBase);
1254 
1255   Instruction *InsertPt = &*F.getEntryBlock().begin();
1256   IRBuilder<> EntryIRB(InsertPt);
1257   emitPrologue(EntryIRB,
1258                /*WithFrameRecord*/ ClRecordStackHistory &&
1259                    !AllocasToInstrument.empty());
1260 
1261   if (!AllocasToInstrument.empty()) {
1262     Value *StackTag =
1263         ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
1264     instrumentStack(AllocasToInstrument, AllocaDbgMap, RetVec, StackTag);
1265   }
1266   // Pad and align each of the allocas that we instrumented to stop small
1267   // uninteresting allocas from hiding in instrumented alloca's padding and so
1268   // that we have enough space to store real tags for short granules.
1269   DenseMap<AllocaInst *, AllocaInst *> AllocaToPaddedAllocaMap;
1270   for (AllocaInst *AI : AllocasToInstrument) {
1271     uint64_t Size = getAllocaSizeInBytes(*AI);
1272     uint64_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
1273     AI->setAlignment(
1274         Align(std::max(AI->getAlignment(), Mapping.getObjectAlignment())));
1275     if (Size != AlignedSize) {
1276       Type *AllocatedType = AI->getAllocatedType();
1277       if (AI->isArrayAllocation()) {
1278         uint64_t ArraySize =
1279             cast<ConstantInt>(AI->getArraySize())->getZExtValue();
1280         AllocatedType = ArrayType::get(AllocatedType, ArraySize);
1281       }
1282       Type *TypeWithPadding = StructType::get(
1283           AllocatedType, ArrayType::get(Int8Ty, AlignedSize - Size));
1284       auto *NewAI = new AllocaInst(
1285           TypeWithPadding, AI->getType()->getAddressSpace(), nullptr, "", AI);
1286       NewAI->takeName(AI);
1287       NewAI->setAlignment(AI->getAlign());
1288       NewAI->setUsedWithInAlloca(AI->isUsedWithInAlloca());
1289       NewAI->setSwiftError(AI->isSwiftError());
1290       NewAI->copyMetadata(*AI);
1291       auto *Bitcast = new BitCastInst(NewAI, AI->getType(), "", AI);
1292       AI->replaceAllUsesWith(Bitcast);
1293       AllocaToPaddedAllocaMap[AI] = NewAI;
1294     }
1295   }
1296 
1297   if (!AllocaToPaddedAllocaMap.empty()) {
1298     for (auto &BB : F)
1299       for (auto &Inst : BB)
1300         if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst))
1301           if (auto *AI =
1302                   dyn_cast_or_null<AllocaInst>(DVI->getVariableLocation()))
1303             if (auto *NewAI = AllocaToPaddedAllocaMap.lookup(AI))
1304               DVI->setArgOperand(
1305                   0, MetadataAsValue::get(*C, LocalAsMetadata::get(NewAI)));
1306     for (auto &P : AllocaToPaddedAllocaMap)
1307       P.first->eraseFromParent();
1308   }
1309 
1310   // If we split the entry block, move any allocas that were originally in the
1311   // entry block back into the entry block so that they aren't treated as
1312   // dynamic allocas.
1313   if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1314     InsertPt = &*F.getEntryBlock().begin();
1315     for (auto II = EntryIRB.GetInsertBlock()->begin(),
1316               IE = EntryIRB.GetInsertBlock()->end();
1317          II != IE;) {
1318       Instruction *I = &*II++;
1319       if (auto *AI = dyn_cast<AllocaInst>(I))
1320         if (isa<ConstantInt>(AI->getArraySize()))
1321           I->moveBefore(InsertPt);
1322     }
1323   }
1324 
1325   for (auto &Operand : OperandsToInstrument)
1326     instrumentMemAccess(Operand);
1327 
1328   if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
1329     for (auto Inst : IntrinToInstrument)
1330       instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
1331   }
1332 
1333   ShadowBase = nullptr;
1334   StackBaseTag = nullptr;
1335 
1336   return true;
1337 }
1338 
1339 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1340   Constant *Initializer = GV->getInitializer();
1341   uint64_t SizeInBytes =
1342       M.getDataLayout().getTypeAllocSize(Initializer->getType());
1343   uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
1344   if (SizeInBytes != NewSize) {
1345     // Pad the initializer out to the next multiple of 16 bytes and add the
1346     // required short granule tag.
1347     std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1348     Init.back() = Tag;
1349     Constant *Padding = ConstantDataArray::get(*C, Init);
1350     Initializer = ConstantStruct::getAnon({Initializer, Padding});
1351   }
1352 
1353   auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1354                                    GlobalValue::ExternalLinkage, Initializer,
1355                                    GV->getName() + ".hwasan");
1356   NewGV->copyAttributesFrom(GV);
1357   NewGV->setLinkage(GlobalValue::PrivateLinkage);
1358   NewGV->copyMetadata(GV, 0);
1359   NewGV->setAlignment(
1360       MaybeAlign(std::max(GV->getAlignment(), Mapping.getObjectAlignment())));
1361 
1362   // It is invalid to ICF two globals that have different tags. In the case
1363   // where the size of the global is a multiple of the tag granularity the
1364   // contents of the globals may be the same but the tags (i.e. symbol values)
1365   // may be different, and the symbols are not considered during ICF. In the
1366   // case where the size is not a multiple of the granularity, the short granule
1367   // tags would discriminate two globals with different tags, but there would
1368   // otherwise be nothing stopping such a global from being incorrectly ICF'd
1369   // with an uninstrumented (i.e. tag 0) global that happened to have the short
1370   // granule tag in the last byte.
1371   NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1372 
1373   // Descriptor format (assuming little-endian):
1374   // bytes 0-3: relative address of global
1375   // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1376   // it isn't, we create multiple descriptors)
1377   // byte 7: tag
1378   auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
1379   const uint64_t MaxDescriptorSize = 0xfffff0;
1380   for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1381        DescriptorPos += MaxDescriptorSize) {
1382     auto *Descriptor =
1383         new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1384                            nullptr, GV->getName() + ".hwasan.descriptor");
1385     auto *GVRelPtr = ConstantExpr::getTrunc(
1386         ConstantExpr::getAdd(
1387             ConstantExpr::getSub(
1388                 ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1389                 ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
1390             ConstantInt::get(Int64Ty, DescriptorPos)),
1391         Int32Ty);
1392     uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
1393     auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
1394     Descriptor->setComdat(NewGV->getComdat());
1395     Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
1396     Descriptor->setSection("hwasan_globals");
1397     Descriptor->setMetadata(LLVMContext::MD_associated,
1398                             MDNode::get(*C, ValueAsMetadata::get(NewGV)));
1399     appendToCompilerUsed(M, Descriptor);
1400   }
1401 
1402   Constant *Aliasee = ConstantExpr::getIntToPtr(
1403       ConstantExpr::getAdd(
1404           ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1405           ConstantInt::get(Int64Ty, uint64_t(Tag) << kPointerTagShift)),
1406       GV->getType());
1407   auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
1408                                     GV->getLinkage(), "", Aliasee, &M);
1409   Alias->setVisibility(GV->getVisibility());
1410   Alias->takeName(GV);
1411   GV->replaceAllUsesWith(Alias);
1412   GV->eraseFromParent();
1413 }
1414 
1415 void HWAddressSanitizer::instrumentGlobals() {
1416   std::vector<GlobalVariable *> Globals;
1417   for (GlobalVariable &GV : M.globals()) {
1418     if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") ||
1419         GV.isThreadLocal())
1420       continue;
1421 
1422     // Common symbols can't have aliases point to them, so they can't be tagged.
1423     if (GV.hasCommonLinkage())
1424       continue;
1425 
1426     // Globals with custom sections may be used in __start_/__stop_ enumeration,
1427     // which would be broken both by adding tags and potentially by the extra
1428     // padding/alignment that we insert.
1429     if (GV.hasSection())
1430       continue;
1431 
1432     Globals.push_back(&GV);
1433   }
1434 
1435   MD5 Hasher;
1436   Hasher.update(M.getSourceFileName());
1437   MD5::MD5Result Hash;
1438   Hasher.final(Hash);
1439   uint8_t Tag = Hash[0];
1440 
1441   for (GlobalVariable *GV : Globals) {
1442     // Skip tag 0 in order to avoid collisions with untagged memory.
1443     if (Tag == 0)
1444       Tag = 1;
1445     instrumentGlobal(GV, Tag++);
1446   }
1447 }
1448 
1449 void HWAddressSanitizer::instrumentPersonalityFunctions() {
1450   // We need to untag stack frames as we unwind past them. That is the job of
1451   // the personality function wrapper, which either wraps an existing
1452   // personality function or acts as a personality function on its own. Each
1453   // function that has a personality function or that can be unwound past has
1454   // its personality function changed to a thunk that calls the personality
1455   // function wrapper in the runtime.
1456   MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1457   for (Function &F : M) {
1458     if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress))
1459       continue;
1460 
1461     if (F.hasPersonalityFn()) {
1462       PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F);
1463     } else if (!F.hasFnAttribute(Attribute::NoUnwind)) {
1464       PersonalityFns[nullptr].push_back(&F);
1465     }
1466   }
1467 
1468   if (PersonalityFns.empty())
1469     return;
1470 
1471   FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1472       "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty,
1473       Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy);
1474   FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy);
1475   FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy);
1476 
1477   for (auto &P : PersonalityFns) {
1478     std::string ThunkName = kHwasanPersonalityThunkName;
1479     if (P.first)
1480       ThunkName += ("." + P.first->getName()).str();
1481     FunctionType *ThunkFnTy = FunctionType::get(
1482         Int32Ty, {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int8PtrTy}, false);
1483     bool IsLocal = P.first && (!isa<GlobalValue>(P.first) ||
1484                                cast<GlobalValue>(P.first)->hasLocalLinkage());
1485     auto *ThunkFn = Function::Create(ThunkFnTy,
1486                                      IsLocal ? GlobalValue::InternalLinkage
1487                                              : GlobalValue::LinkOnceODRLinkage,
1488                                      ThunkName, &M);
1489     if (!IsLocal) {
1490       ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1491       ThunkFn->setComdat(M.getOrInsertComdat(ThunkName));
1492     }
1493 
1494     auto *BB = BasicBlock::Create(*C, "entry", ThunkFn);
1495     IRBuilder<> IRB(BB);
1496     CallInst *WrapperCall = IRB.CreateCall(
1497         HwasanPersonalityWrapper,
1498         {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2),
1499          ThunkFn->getArg(3), ThunkFn->getArg(4),
1500          P.first ? IRB.CreateBitCast(P.first, Int8PtrTy)
1501                  : Constant::getNullValue(Int8PtrTy),
1502          IRB.CreateBitCast(UnwindGetGR.getCallee(), Int8PtrTy),
1503          IRB.CreateBitCast(UnwindGetCFA.getCallee(), Int8PtrTy)});
1504     WrapperCall->setTailCall();
1505     IRB.CreateRet(WrapperCall);
1506 
1507     for (Function *F : P.second)
1508       F->setPersonalityFn(ThunkFn);
1509   }
1510 }
1511 
1512 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
1513   Scale = kDefaultShadowScale;
1514   if (ClMappingOffset.getNumOccurrences() > 0) {
1515     InGlobal = false;
1516     InTls = false;
1517     Offset = ClMappingOffset;
1518   } else if (ClEnableKhwasan || ClInstrumentWithCalls) {
1519     InGlobal = false;
1520     InTls = false;
1521     Offset = 0;
1522   } else if (ClWithIfunc) {
1523     InGlobal = true;
1524     InTls = false;
1525     Offset = kDynamicShadowSentinel;
1526   } else if (ClWithTls) {
1527     InGlobal = false;
1528     InTls = true;
1529     Offset = kDynamicShadowSentinel;
1530   } else {
1531     InGlobal = false;
1532     InTls = false;
1533     Offset = kDynamicShadowSentinel;
1534   }
1535 }
1536