1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file is a part of HWAddressSanitizer, an address sanity checker
11 /// based on tagged addressing.
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15 #include "llvm/ADT/MapVector.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/StringExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/BinaryFormat/ELF.h"
21 #include "llvm/IR/Attributes.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/Constant.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DebugInfoMetadata.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/IRBuilder.h"
30 #include "llvm/IR/InlineAsm.h"
31 #include "llvm/IR/InstVisitor.h"
32 #include "llvm/IR/Instruction.h"
33 #include "llvm/IR/Instructions.h"
34 #include "llvm/IR/IntrinsicInst.h"
35 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/IR/LLVMContext.h"
37 #include "llvm/IR/MDBuilder.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/IR/Value.h"
41 #include "llvm/InitializePasses.h"
42 #include "llvm/Pass.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/CommandLine.h"
45 #include "llvm/Support/Debug.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Transforms/Instrumentation.h"
48 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
49 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
50 #include "llvm/Transforms/Utils/ModuleUtils.h"
51 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
52 #include <sstream>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "hwasan"
57 
58 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
59 static const char *const kHwasanNoteName = "hwasan.note";
60 static const char *const kHwasanInitName = "__hwasan_init";
61 static const char *const kHwasanPersonalityThunkName =
62     "__hwasan_personality_thunk";
63 
64 static const char *const kHwasanShadowMemoryDynamicAddress =
65     "__hwasan_shadow_memory_dynamic_address";
66 
67 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
68 static const size_t kNumberOfAccessSizes = 5;
69 
70 static const size_t kDefaultShadowScale = 4;
71 static const uint64_t kDynamicShadowSentinel =
72     std::numeric_limits<uint64_t>::max();
73 static const unsigned kPointerTagShift = 56;
74 
75 static const unsigned kShadowBaseAlignment = 32;
76 
77 static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
78     "hwasan-memory-access-callback-prefix",
79     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
80     cl::init("__hwasan_"));
81 
82 static cl::opt<bool>
83     ClInstrumentWithCalls("hwasan-instrument-with-calls",
84                 cl::desc("instrument reads and writes with callbacks"),
85                 cl::Hidden, cl::init(false));
86 
87 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
88                                        cl::desc("instrument read instructions"),
89                                        cl::Hidden, cl::init(true));
90 
91 static cl::opt<bool> ClInstrumentWrites(
92     "hwasan-instrument-writes", cl::desc("instrument write instructions"),
93     cl::Hidden, cl::init(true));
94 
95 static cl::opt<bool> ClInstrumentAtomics(
96     "hwasan-instrument-atomics",
97     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
98     cl::init(true));
99 
100 static cl::opt<bool> ClInstrumentByval("hwasan-instrument-byval",
101                                        cl::desc("instrument byval arguments"),
102                                        cl::Hidden, cl::init(true));
103 
104 static cl::opt<bool> ClRecover(
105     "hwasan-recover",
106     cl::desc("Enable recovery mode (continue-after-error)."),
107     cl::Hidden, cl::init(false));
108 
109 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
110                                        cl::desc("instrument stack (allocas)"),
111                                        cl::Hidden, cl::init(true));
112 
113 static cl::opt<bool> ClUARRetagToZero(
114     "hwasan-uar-retag-to-zero",
115     cl::desc("Clear alloca tags before returning from the function to allow "
116              "non-instrumented and instrumented function calls mix. When set "
117              "to false, allocas are retagged before returning from the "
118              "function to detect use after return."),
119     cl::Hidden, cl::init(true));
120 
121 static cl::opt<bool> ClGenerateTagsWithCalls(
122     "hwasan-generate-tags-with-calls",
123     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
124     cl::init(false));
125 
126 static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
127                                cl::Hidden, cl::init(false), cl::ZeroOrMore);
128 
129 static cl::opt<int> ClMatchAllTag(
130     "hwasan-match-all-tag",
131     cl::desc("don't report bad accesses via pointers with this tag"),
132     cl::Hidden, cl::init(-1));
133 
134 static cl::opt<bool> ClEnableKhwasan(
135     "hwasan-kernel",
136     cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
137     cl::Hidden, cl::init(false));
138 
139 // These flags allow to change the shadow mapping and control how shadow memory
140 // is accessed. The shadow mapping looks like:
141 //    Shadow = (Mem >> scale) + offset
142 
143 static cl::opt<uint64_t>
144     ClMappingOffset("hwasan-mapping-offset",
145                     cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
146                     cl::Hidden, cl::init(0));
147 
148 static cl::opt<bool>
149     ClWithIfunc("hwasan-with-ifunc",
150                 cl::desc("Access dynamic shadow through an ifunc global on "
151                          "platforms that support this"),
152                 cl::Hidden, cl::init(false));
153 
154 static cl::opt<bool> ClWithTls(
155     "hwasan-with-tls",
156     cl::desc("Access dynamic shadow through an thread-local pointer on "
157              "platforms that support this"),
158     cl::Hidden, cl::init(true));
159 
160 static cl::opt<bool>
161     ClRecordStackHistory("hwasan-record-stack-history",
162                          cl::desc("Record stack frames with tagged allocations "
163                                   "in a thread-local ring buffer"),
164                          cl::Hidden, cl::init(true));
165 static cl::opt<bool>
166     ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
167                               cl::desc("instrument memory intrinsics"),
168                               cl::Hidden, cl::init(true));
169 
170 static cl::opt<bool>
171     ClInstrumentLandingPads("hwasan-instrument-landing-pads",
172                             cl::desc("instrument landing pads"), cl::Hidden,
173                             cl::init(false), cl::ZeroOrMore);
174 
175 static cl::opt<bool> ClUseShortGranules(
176     "hwasan-use-short-granules",
177     cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
178     cl::init(false), cl::ZeroOrMore);
179 
180 static cl::opt<bool> ClInstrumentPersonalityFunctions(
181     "hwasan-instrument-personality-functions",
182     cl::desc("instrument personality functions"), cl::Hidden, cl::init(false),
183     cl::ZeroOrMore);
184 
185 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
186                                        cl::desc("inline all checks"),
187                                        cl::Hidden, cl::init(false));
188 
189 namespace {
190 
191 /// An instrumentation pass implementing detection of addressability bugs
192 /// using tagged pointers.
193 class HWAddressSanitizer {
194 public:
195   explicit HWAddressSanitizer(Module &M, bool CompileKernel = false,
196                               bool Recover = false) : M(M) {
197     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
198     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
199         ClEnableKhwasan : CompileKernel;
200 
201     initializeModule();
202   }
203 
204   bool sanitizeFunction(Function &F);
205   void initializeModule();
206   void createHwasanCtorComdat();
207 
208   void initializeCallbacks(Module &M);
209 
210   Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
211 
212   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
213   Value *getShadowNonTls(IRBuilder<> &IRB);
214 
215   void untagPointerOperand(Instruction *I, Value *Addr);
216   Value *shadowBase();
217   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
218   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
219                                  unsigned AccessSizeIndex,
220                                  Instruction *InsertBefore);
221   void instrumentMemIntrinsic(MemIntrinsic *MI);
222   bool instrumentMemAccess(InterestingMemoryOperand &O);
223   bool ignoreAccess(Value *Ptr);
224   void getInterestingMemoryOperands(
225       Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
226 
227   bool isInterestingAlloca(const AllocaInst &AI);
228   bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
229   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
230   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
231   bool instrumentStack(
232       SmallVectorImpl<AllocaInst *> &Allocas,
233       DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap,
234       SmallVectorImpl<Instruction *> &RetVec, Value *StackTag);
235   Value *readRegister(IRBuilder<> &IRB, StringRef Name);
236   bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
237   Value *getNextTagWithCall(IRBuilder<> &IRB);
238   Value *getStackBaseTag(IRBuilder<> &IRB);
239   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
240                      unsigned AllocaNo);
241   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
242 
243   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
244   void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
245 
246   void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
247   void instrumentGlobals();
248 
249   void instrumentPersonalityFunctions();
250 
251 private:
252   LLVMContext *C;
253   Module &M;
254   Triple TargetTriple;
255   FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
256   FunctionCallee HWAsanHandleVfork;
257 
258   /// This struct defines the shadow mapping using the rule:
259   ///   shadow = (mem >> Scale) + Offset.
260   /// If InGlobal is true, then
261   ///   extern char __hwasan_shadow[];
262   ///   shadow = (mem >> Scale) + &__hwasan_shadow
263   /// If InTls is true, then
264   ///   extern char *__hwasan_tls;
265   ///   shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
266   struct ShadowMapping {
267     int Scale;
268     uint64_t Offset;
269     bool InGlobal;
270     bool InTls;
271 
272     void init(Triple &TargetTriple);
273     unsigned getObjectAlignment() const { return 1U << Scale; }
274   };
275   ShadowMapping Mapping;
276 
277   Type *VoidTy = Type::getVoidTy(M.getContext());
278   Type *IntptrTy;
279   Type *Int8PtrTy;
280   Type *Int8Ty;
281   Type *Int32Ty;
282   Type *Int64Ty = Type::getInt64Ty(M.getContext());
283 
284   bool CompileKernel;
285   bool Recover;
286   bool OutlinedChecks;
287   bool UseShortGranules;
288   bool InstrumentLandingPads;
289 
290   bool HasMatchAllTag = false;
291   uint8_t MatchAllTag = 0;
292 
293   Function *HwasanCtorFunction;
294 
295   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
296   FunctionCallee HwasanMemoryAccessCallbackSized[2];
297 
298   FunctionCallee HwasanTagMemoryFunc;
299   FunctionCallee HwasanGenerateTagFunc;
300 
301   Constant *ShadowGlobal;
302 
303   Value *ShadowBase = nullptr;
304   Value *StackBaseTag = nullptr;
305   GlobalValue *ThreadPtrGlobal = nullptr;
306 };
307 
308 class HWAddressSanitizerLegacyPass : public FunctionPass {
309 public:
310   // Pass identification, replacement for typeid.
311   static char ID;
312 
313   explicit HWAddressSanitizerLegacyPass(bool CompileKernel = false,
314                                         bool Recover = false)
315       : FunctionPass(ID), CompileKernel(CompileKernel), Recover(Recover) {
316     initializeHWAddressSanitizerLegacyPassPass(
317         *PassRegistry::getPassRegistry());
318   }
319 
320   StringRef getPassName() const override { return "HWAddressSanitizer"; }
321 
322   bool doInitialization(Module &M) override {
323     HWASan = std::make_unique<HWAddressSanitizer>(M, CompileKernel, Recover);
324     return true;
325   }
326 
327   bool runOnFunction(Function &F) override {
328     return HWASan->sanitizeFunction(F);
329   }
330 
331   bool doFinalization(Module &M) override {
332     HWASan.reset();
333     return false;
334   }
335 
336 private:
337   std::unique_ptr<HWAddressSanitizer> HWASan;
338   bool CompileKernel;
339   bool Recover;
340 };
341 
342 } // end anonymous namespace
343 
344 char HWAddressSanitizerLegacyPass::ID = 0;
345 
346 INITIALIZE_PASS_BEGIN(
347     HWAddressSanitizerLegacyPass, "hwasan",
348     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
349     false)
350 INITIALIZE_PASS_END(
351     HWAddressSanitizerLegacyPass, "hwasan",
352     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
353     false)
354 
355 FunctionPass *llvm::createHWAddressSanitizerLegacyPassPass(bool CompileKernel,
356                                                            bool Recover) {
357   assert(!CompileKernel || Recover);
358   return new HWAddressSanitizerLegacyPass(CompileKernel, Recover);
359 }
360 
361 HWAddressSanitizerPass::HWAddressSanitizerPass(bool CompileKernel, bool Recover)
362     : CompileKernel(CompileKernel), Recover(Recover) {}
363 
364 PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
365                                               ModuleAnalysisManager &MAM) {
366   HWAddressSanitizer HWASan(M, CompileKernel, Recover);
367   bool Modified = false;
368   for (Function &F : M)
369     Modified |= HWASan.sanitizeFunction(F);
370   if (Modified)
371     return PreservedAnalyses::none();
372   return PreservedAnalyses::all();
373 }
374 
375 void HWAddressSanitizer::createHwasanCtorComdat() {
376   std::tie(HwasanCtorFunction, std::ignore) =
377       getOrCreateSanitizerCtorAndInitFunctions(
378           M, kHwasanModuleCtorName, kHwasanInitName,
379           /*InitArgTypes=*/{},
380           /*InitArgs=*/{},
381           // This callback is invoked when the functions are created the first
382           // time. Hook them into the global ctors list in that case:
383           [&](Function *Ctor, FunctionCallee) {
384             Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
385             Ctor->setComdat(CtorComdat);
386             appendToGlobalCtors(M, Ctor, 0, Ctor);
387           });
388 
389   // Create a note that contains pointers to the list of global
390   // descriptors. Adding a note to the output file will cause the linker to
391   // create a PT_NOTE program header pointing to the note that we can use to
392   // find the descriptor list starting from the program headers. A function
393   // provided by the runtime initializes the shadow memory for the globals by
394   // accessing the descriptor list via the note. The dynamic loader needs to
395   // call this function whenever a library is loaded.
396   //
397   // The reason why we use a note for this instead of a more conventional
398   // approach of having a global constructor pass a descriptor list pointer to
399   // the runtime is because of an order of initialization problem. With
400   // constructors we can encounter the following problematic scenario:
401   //
402   // 1) library A depends on library B and also interposes one of B's symbols
403   // 2) B's constructors are called before A's (as required for correctness)
404   // 3) during construction, B accesses one of its "own" globals (actually
405   //    interposed by A) and triggers a HWASAN failure due to the initialization
406   //    for A not having happened yet
407   //
408   // Even without interposition it is possible to run into similar situations in
409   // cases where two libraries mutually depend on each other.
410   //
411   // We only need one note per binary, so put everything for the note in a
412   // comdat. This needs to be a comdat with an .init_array section to prevent
413   // newer versions of lld from discarding the note.
414   //
415   // Create the note even if we aren't instrumenting globals. This ensures that
416   // binaries linked from object files with both instrumented and
417   // non-instrumented globals will end up with a note, even if a comdat from an
418   // object file with non-instrumented globals is selected. The note is harmless
419   // if the runtime doesn't support it, since it will just be ignored.
420   Comdat *NoteComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
421 
422   Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
423   auto Start =
424       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
425                          nullptr, "__start_hwasan_globals");
426   Start->setVisibility(GlobalValue::HiddenVisibility);
427   Start->setDSOLocal(true);
428   auto Stop =
429       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
430                          nullptr, "__stop_hwasan_globals");
431   Stop->setVisibility(GlobalValue::HiddenVisibility);
432   Stop->setDSOLocal(true);
433 
434   // Null-terminated so actually 8 bytes, which are required in order to align
435   // the note properly.
436   auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
437 
438   auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
439                                  Int32Ty, Int32Ty);
440   auto *Note =
441       new GlobalVariable(M, NoteTy, /*isConstant=*/true,
442                          GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
443   Note->setSection(".note.hwasan.globals");
444   Note->setComdat(NoteComdat);
445   Note->setAlignment(Align(4));
446   Note->setDSOLocal(true);
447 
448   // The pointers in the note need to be relative so that the note ends up being
449   // placed in rodata, which is the standard location for notes.
450   auto CreateRelPtr = [&](Constant *Ptr) {
451     return ConstantExpr::getTrunc(
452         ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
453                              ConstantExpr::getPtrToInt(Note, Int64Ty)),
454         Int32Ty);
455   };
456   Note->setInitializer(ConstantStruct::getAnon(
457       {ConstantInt::get(Int32Ty, 8),                           // n_namesz
458        ConstantInt::get(Int32Ty, 8),                           // n_descsz
459        ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
460        Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
461   appendToCompilerUsed(M, Note);
462 
463   // Create a zero-length global in hwasan_globals so that the linker will
464   // always create start and stop symbols.
465   auto Dummy = new GlobalVariable(
466       M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
467       Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
468   Dummy->setSection("hwasan_globals");
469   Dummy->setComdat(NoteComdat);
470   Dummy->setMetadata(LLVMContext::MD_associated,
471                      MDNode::get(*C, ValueAsMetadata::get(Note)));
472   appendToCompilerUsed(M, Dummy);
473 }
474 
475 /// Module-level initialization.
476 ///
477 /// inserts a call to __hwasan_init to the module's constructor list.
478 void HWAddressSanitizer::initializeModule() {
479   LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
480   auto &DL = M.getDataLayout();
481 
482   TargetTriple = Triple(M.getTargetTriple());
483 
484   Mapping.init(TargetTriple);
485 
486   C = &(M.getContext());
487   IRBuilder<> IRB(*C);
488   IntptrTy = IRB.getIntPtrTy(DL);
489   Int8PtrTy = IRB.getInt8PtrTy();
490   Int8Ty = IRB.getInt8Ty();
491   Int32Ty = IRB.getInt32Ty();
492 
493   HwasanCtorFunction = nullptr;
494 
495   // Older versions of Android do not have the required runtime support for
496   // short granules, global or personality function instrumentation. On other
497   // platforms we currently require using the latest version of the runtime.
498   bool NewRuntime =
499       !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
500 
501   UseShortGranules =
502       ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
503   OutlinedChecks =
504       TargetTriple.isAArch64() && TargetTriple.isOSBinFormatELF() &&
505       (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
506 
507   if (ClMatchAllTag.getNumOccurrences()) {
508     if (ClMatchAllTag != -1) {
509       HasMatchAllTag = true;
510       MatchAllTag = ClMatchAllTag & 0xFF;
511     }
512   } else if (CompileKernel) {
513     HasMatchAllTag = true;
514     MatchAllTag = 0xFF;
515   }
516 
517   // If we don't have personality function support, fall back to landing pads.
518   InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
519                               ? ClInstrumentLandingPads
520                               : !NewRuntime;
521 
522   if (!CompileKernel) {
523     createHwasanCtorComdat();
524     bool InstrumentGlobals =
525         ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
526     if (InstrumentGlobals)
527       instrumentGlobals();
528 
529     bool InstrumentPersonalityFunctions =
530         ClInstrumentPersonalityFunctions.getNumOccurrences()
531             ? ClInstrumentPersonalityFunctions
532             : NewRuntime;
533     if (InstrumentPersonalityFunctions)
534       instrumentPersonalityFunctions();
535   }
536 
537   if (!TargetTriple.isAndroid()) {
538     Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
539       auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
540                                     GlobalValue::ExternalLinkage, nullptr,
541                                     "__hwasan_tls", nullptr,
542                                     GlobalVariable::InitialExecTLSModel);
543       appendToCompilerUsed(M, GV);
544       return GV;
545     });
546     ThreadPtrGlobal = cast<GlobalVariable>(C);
547   }
548 }
549 
550 void HWAddressSanitizer::initializeCallbacks(Module &M) {
551   IRBuilder<> IRB(*C);
552   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
553     const std::string TypeStr = AccessIsWrite ? "store" : "load";
554     const std::string EndingStr = Recover ? "_noabort" : "";
555 
556     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
557         ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
558         FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
559 
560     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
561          AccessSizeIndex++) {
562       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
563           M.getOrInsertFunction(
564               ClMemoryAccessCallbackPrefix + TypeStr +
565                   itostr(1ULL << AccessSizeIndex) + EndingStr,
566               FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
567     }
568   }
569 
570   HwasanTagMemoryFunc = M.getOrInsertFunction(
571       "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
572   HwasanGenerateTagFunc =
573       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
574 
575   ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
576                                      ArrayType::get(IRB.getInt8Ty(), 0));
577 
578   const std::string MemIntrinCallbackPrefix =
579       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
580   HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
581                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
582                                         IRB.getInt8PtrTy(), IntptrTy);
583   HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
584                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
585                                        IRB.getInt8PtrTy(), IntptrTy);
586   HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
587                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
588                                        IRB.getInt32Ty(), IntptrTy);
589 
590   HWAsanHandleVfork =
591       M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy);
592 }
593 
594 Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
595   // An empty inline asm with input reg == output reg.
596   // An opaque no-op cast, basically.
597   // This prevents code bloat as a result of rematerializing trivial definitions
598   // such as constants or global addresses at every load and store.
599   InlineAsm *Asm =
600       InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
601                      StringRef(""), StringRef("=r,0"),
602                      /*hasSideEffects=*/false);
603   return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
604 }
605 
606 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
607   return getOpaqueNoopCast(IRB, ShadowGlobal);
608 }
609 
610 Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
611   if (Mapping.Offset != kDynamicShadowSentinel)
612     return getOpaqueNoopCast(
613         IRB, ConstantExpr::getIntToPtr(
614                  ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy));
615 
616   if (Mapping.InGlobal) {
617     return getDynamicShadowIfunc(IRB);
618   } else {
619     Value *GlobalDynamicAddress =
620         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
621             kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
622     return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
623   }
624 }
625 
626 bool HWAddressSanitizer::ignoreAccess(Value *Ptr) {
627   // Do not instrument acesses from different address spaces; we cannot deal
628   // with them.
629   Type *PtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
630   if (PtrTy->getPointerAddressSpace() != 0)
631     return true;
632 
633   // Ignore swifterror addresses.
634   // swifterror memory addresses are mem2reg promoted by instruction
635   // selection. As such they cannot have regular uses like an instrumentation
636   // function and it makes no sense to track them as memory.
637   if (Ptr->isSwiftError())
638     return true;
639 
640   return false;
641 }
642 
643 void HWAddressSanitizer::getInterestingMemoryOperands(
644     Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
645   // Skip memory accesses inserted by another instrumentation.
646   if (I->hasMetadata("nosanitize"))
647     return;
648 
649   // Do not instrument the load fetching the dynamic shadow address.
650   if (ShadowBase == I)
651     return;
652 
653   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
654     if (!ClInstrumentReads || ignoreAccess(LI->getPointerOperand()))
655       return;
656     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
657                              LI->getType(), LI->getAlign());
658   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
659     if (!ClInstrumentWrites || ignoreAccess(SI->getPointerOperand()))
660       return;
661     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
662                              SI->getValueOperand()->getType(), SI->getAlign());
663   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
664     if (!ClInstrumentAtomics || ignoreAccess(RMW->getPointerOperand()))
665       return;
666     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
667                              RMW->getValOperand()->getType(), None);
668   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
669     if (!ClInstrumentAtomics || ignoreAccess(XCHG->getPointerOperand()))
670       return;
671     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
672                              XCHG->getCompareOperand()->getType(), None);
673   } else if (auto CI = dyn_cast<CallInst>(I)) {
674     for (unsigned ArgNo = 0; ArgNo < CI->getNumArgOperands(); ArgNo++) {
675       if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
676           ignoreAccess(CI->getArgOperand(ArgNo)))
677         continue;
678       Type *Ty = CI->getParamByValType(ArgNo);
679       Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
680     }
681   }
682 }
683 
684 static unsigned getPointerOperandIndex(Instruction *I) {
685   if (LoadInst *LI = dyn_cast<LoadInst>(I))
686     return LI->getPointerOperandIndex();
687   if (StoreInst *SI = dyn_cast<StoreInst>(I))
688     return SI->getPointerOperandIndex();
689   if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
690     return RMW->getPointerOperandIndex();
691   if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
692     return XCHG->getPointerOperandIndex();
693   report_fatal_error("Unexpected instruction");
694   return -1;
695 }
696 
697 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
698   size_t Res = countTrailingZeros(TypeSize / 8);
699   assert(Res < kNumberOfAccessSizes);
700   return Res;
701 }
702 
703 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
704   if (TargetTriple.isAArch64())
705     return;
706 
707   IRBuilder<> IRB(I);
708   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
709   Value *UntaggedPtr =
710       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
711   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
712 }
713 
714 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
715   // Mem >> Scale
716   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
717   if (Mapping.Offset == 0)
718     return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
719   // (Mem >> Scale) + Offset
720   return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow);
721 }
722 
723 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
724                                                    unsigned AccessSizeIndex,
725                                                    Instruction *InsertBefore) {
726   const int64_t AccessInfo =
727       (CompileKernel << HWASanAccessInfo::CompileKernelShift) +
728       (HasMatchAllTag << HWASanAccessInfo::HasMatchAllShift) +
729       (MatchAllTag << HWASanAccessInfo::MatchAllShift) +
730       (Recover << HWASanAccessInfo::RecoverShift) +
731       (IsWrite << HWASanAccessInfo::IsWriteShift) +
732       (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
733   IRBuilder<> IRB(InsertBefore);
734 
735   if (OutlinedChecks) {
736     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
737     Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
738     IRB.CreateCall(Intrinsic::getDeclaration(
739                        M, UseShortGranules
740                               ? Intrinsic::hwasan_check_memaccess_shortgranules
741                               : Intrinsic::hwasan_check_memaccess),
742                    {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
743     return;
744   }
745 
746   Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
747   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
748                                   IRB.getInt8Ty());
749   Value *AddrLong = untagPointer(IRB, PtrLong);
750   Value *Shadow = memToShadow(AddrLong, IRB);
751   Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
752   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
753 
754   if (HasMatchAllTag) {
755     Value *TagNotIgnored = IRB.CreateICmpNE(
756         PtrTag, ConstantInt::get(PtrTag->getType(), MatchAllTag));
757     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
758   }
759 
760   Instruction *CheckTerm =
761       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
762                                 MDBuilder(*C).createBranchWeights(1, 100000));
763 
764   IRB.SetInsertPoint(CheckTerm);
765   Value *OutOfShortGranuleTagRange =
766       IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15));
767   Instruction *CheckFailTerm =
768       SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover,
769                                 MDBuilder(*C).createBranchWeights(1, 100000));
770 
771   IRB.SetInsertPoint(CheckTerm);
772   Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty);
773   PtrLowBits = IRB.CreateAdd(
774       PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1));
775   Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag);
776   SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false,
777                             MDBuilder(*C).createBranchWeights(1, 100000),
778                             nullptr, nullptr, CheckFailTerm->getParent());
779 
780   IRB.SetInsertPoint(CheckTerm);
781   Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15);
782   InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy);
783   Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);
784   Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag);
785   SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false,
786                             MDBuilder(*C).createBranchWeights(1, 100000),
787                             nullptr, nullptr, CheckFailTerm->getParent());
788 
789   IRB.SetInsertPoint(CheckFailTerm);
790   InlineAsm *Asm;
791   switch (TargetTriple.getArch()) {
792     case Triple::x86_64:
793       // The signal handler will find the data address in rdi.
794       Asm = InlineAsm::get(
795           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
796           "int3\nnopl " +
797               itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
798               "(%rax)",
799           "{rdi}",
800           /*hasSideEffects=*/true);
801       break;
802     case Triple::aarch64:
803     case Triple::aarch64_be:
804       // The signal handler will find the data address in x0.
805       Asm = InlineAsm::get(
806           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
807           "brk #" +
808               itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
809           "{x0}",
810           /*hasSideEffects=*/true);
811       break;
812     default:
813       report_fatal_error("unsupported architecture");
814   }
815   IRB.CreateCall(Asm, PtrLong);
816   if (Recover)
817     cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent());
818 }
819 
820 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
821   IRBuilder<> IRB(MI);
822   if (isa<MemTransferInst>(MI)) {
823     IRB.CreateCall(
824         isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
825         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
826          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
827          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
828   } else if (isa<MemSetInst>(MI)) {
829     IRB.CreateCall(
830         HWAsanMemset,
831         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
832          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
833          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
834   }
835   MI->eraseFromParent();
836 }
837 
838 bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
839   Value *Addr = O.getPtr();
840 
841   LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
842 
843   if (O.MaybeMask)
844     return false; //FIXME
845 
846   IRBuilder<> IRB(O.getInsn());
847   if (isPowerOf2_64(O.TypeSize) &&
848       (O.TypeSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
849       (!O.Alignment || *O.Alignment >= (1ULL << Mapping.Scale) ||
850        *O.Alignment >= O.TypeSize / 8)) {
851     size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeSize);
852     if (ClInstrumentWithCalls) {
853       IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
854                      IRB.CreatePointerCast(Addr, IntptrTy));
855     } else {
856       instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
857     }
858   } else {
859     IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite],
860                    {IRB.CreatePointerCast(Addr, IntptrTy),
861                     ConstantInt::get(IntptrTy, O.TypeSize / 8)});
862   }
863   untagPointerOperand(O.getInsn(), Addr);
864 
865   return true;
866 }
867 
868 static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
869   uint64_t ArraySize = 1;
870   if (AI.isArrayAllocation()) {
871     const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize());
872     assert(CI && "non-constant array size");
873     ArraySize = CI->getZExtValue();
874   }
875   Type *Ty = AI.getAllocatedType();
876   uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
877   return SizeInBytes * ArraySize;
878 }
879 
880 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
881                                    Value *Tag, size_t Size) {
882   size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
883   if (!UseShortGranules)
884     Size = AlignedSize;
885 
886   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
887   if (ClInstrumentWithCalls) {
888     IRB.CreateCall(HwasanTagMemoryFunc,
889                    {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
890                     ConstantInt::get(IntptrTy, AlignedSize)});
891   } else {
892     size_t ShadowSize = Size >> Mapping.Scale;
893     Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
894     // If this memset is not inlined, it will be intercepted in the hwasan
895     // runtime library. That's OK, because the interceptor skips the checks if
896     // the address is in the shadow region.
897     // FIXME: the interceptor is not as fast as real memset. Consider lowering
898     // llvm.memset right here into either a sequence of stores, or a call to
899     // hwasan_tag_memory.
900     if (ShadowSize)
901       IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, Align(1));
902     if (Size != AlignedSize) {
903       IRB.CreateStore(
904           ConstantInt::get(Int8Ty, Size % Mapping.getObjectAlignment()),
905           IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
906       IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32(
907                                    Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy),
908                                    AlignedSize - 1));
909     }
910   }
911   return true;
912 }
913 
914 static unsigned RetagMask(unsigned AllocaNo) {
915   // A list of 8-bit numbers that have at most one run of non-zero bits.
916   // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
917   // masks.
918   // The list does not include the value 255, which is used for UAR.
919   //
920   // Because we are more likely to use earlier elements of this list than later
921   // ones, it is sorted in increasing order of probability of collision with a
922   // mask allocated (temporally) nearby. The program that generated this list
923   // can be found at:
924   // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
925   static unsigned FastMasks[] = {0,  128, 64,  192, 32,  96,  224, 112, 240,
926                                  48, 16,  120, 248, 56,  24,  8,   124, 252,
927                                  60, 28,  12,  4,   126, 254, 62,  30,  14,
928                                  6,  2,   127, 63,  31,  15,  7,   3,   1};
929   return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))];
930 }
931 
932 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
933   return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
934 }
935 
936 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
937   if (ClGenerateTagsWithCalls)
938     return getNextTagWithCall(IRB);
939   if (StackBaseTag)
940     return StackBaseTag;
941   // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
942   // first).
943   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
944   auto GetStackPointerFn = Intrinsic::getDeclaration(
945       M, Intrinsic::frameaddress,
946       IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
947   Value *StackPointer = IRB.CreateCall(
948       GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())});
949 
950   // Extract some entropy from the stack pointer for the tags.
951   // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
952   // between functions).
953   Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy);
954   Value *StackTag =
955       IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20),
956                     "hwasan.stack.base.tag");
957   return StackTag;
958 }
959 
960 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
961                                         AllocaInst *AI, unsigned AllocaNo) {
962   if (ClGenerateTagsWithCalls)
963     return getNextTagWithCall(IRB);
964   return IRB.CreateXor(StackTag,
965                        ConstantInt::get(IntptrTy, RetagMask(AllocaNo)));
966 }
967 
968 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
969   if (ClUARRetagToZero)
970     return ConstantInt::get(IntptrTy, 0);
971   if (ClGenerateTagsWithCalls)
972     return getNextTagWithCall(IRB);
973   return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
974 }
975 
976 // Add a tag to an address.
977 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
978                                       Value *PtrLong, Value *Tag) {
979   Value *TaggedPtrLong;
980   if (CompileKernel) {
981     // Kernel addresses have 0xFF in the most significant byte.
982     Value *ShiftedTag = IRB.CreateOr(
983         IRB.CreateShl(Tag, kPointerTagShift),
984         ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
985     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
986   } else {
987     // Userspace can simply do OR (tag << 56);
988     Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
989     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
990   }
991   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
992 }
993 
994 // Remove tag from an address.
995 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
996   Value *UntaggedPtrLong;
997   if (CompileKernel) {
998     // Kernel addresses have 0xFF in the most significant byte.
999     UntaggedPtrLong = IRB.CreateOr(PtrLong,
1000         ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
1001   } else {
1002     // Userspace addresses have 0x00.
1003     UntaggedPtrLong = IRB.CreateAnd(PtrLong,
1004         ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
1005   }
1006   return UntaggedPtrLong;
1007 }
1008 
1009 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
1010   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1011   if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
1012     // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1013     // in Bionic's libc/private/bionic_tls.h.
1014     Function *ThreadPointerFunc =
1015         Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
1016     Value *SlotPtr = IRB.CreatePointerCast(
1017         IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
1018                                IRB.CreateCall(ThreadPointerFunc), 0x30),
1019         Ty->getPointerTo(0));
1020     return SlotPtr;
1021   }
1022   if (ThreadPtrGlobal)
1023     return ThreadPtrGlobal;
1024 
1025 
1026   return nullptr;
1027 }
1028 
1029 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1030   if (!Mapping.InTls) {
1031     ShadowBase = getShadowNonTls(IRB);
1032     return;
1033   }
1034 
1035   if (!WithFrameRecord && TargetTriple.isAndroid()) {
1036     ShadowBase = getDynamicShadowIfunc(IRB);
1037     return;
1038   }
1039 
1040   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
1041   assert(SlotPtr);
1042 
1043   Value *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
1044   // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
1045   Value *ThreadLongMaybeUntagged =
1046       TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong);
1047 
1048   if (WithFrameRecord) {
1049     Function *F = IRB.GetInsertBlock()->getParent();
1050     StackBaseTag = IRB.CreateAShr(ThreadLong, 3);
1051 
1052     // Prepare ring buffer data.
1053     Value *PC;
1054     if (TargetTriple.getArch() == Triple::aarch64)
1055       PC = readRegister(IRB, "pc");
1056     else
1057       PC = IRB.CreatePtrToInt(F, IntptrTy);
1058     Module *M = F->getParent();
1059     auto GetStackPointerFn = Intrinsic::getDeclaration(
1060         M, Intrinsic::frameaddress,
1061         IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
1062     Value *SP = IRB.CreatePtrToInt(
1063         IRB.CreateCall(GetStackPointerFn,
1064                        {Constant::getNullValue(IRB.getInt32Ty())}),
1065         IntptrTy);
1066     // Mix SP and PC.
1067     // Assumptions:
1068     // PC is 0x0000PPPPPPPPPPPP  (48 bits are meaningful, others are zero)
1069     // SP is 0xsssssssssssSSSS0  (4 lower bits are zero)
1070     // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
1071     //       0xSSSSPPPPPPPPPPPP
1072     SP = IRB.CreateShl(SP, 44);
1073 
1074     // Store data to ring buffer.
1075     Value *RecordPtr =
1076         IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0));
1077     IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr);
1078 
1079     // Update the ring buffer. Top byte of ThreadLong defines the size of the
1080     // buffer in pages, it must be a power of two, and the start of the buffer
1081     // must be aligned by twice that much. Therefore wrap around of the ring
1082     // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
1083     // The use of AShr instead of LShr is due to
1084     //   https://bugs.llvm.org/show_bug.cgi?id=39030
1085     // Runtime library makes sure not to use the highest bit.
1086     Value *WrapMask = IRB.CreateXor(
1087         IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
1088         ConstantInt::get(IntptrTy, (uint64_t)-1));
1089     Value *ThreadLongNew = IRB.CreateAnd(
1090         IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
1091     IRB.CreateStore(ThreadLongNew, SlotPtr);
1092   }
1093 
1094   // Get shadow base address by aligning RecordPtr up.
1095   // Note: this is not correct if the pointer is already aligned.
1096   // Runtime library will make sure this never happens.
1097   ShadowBase = IRB.CreateAdd(
1098       IRB.CreateOr(
1099           ThreadLongMaybeUntagged,
1100           ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
1101       ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
1102   ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
1103 }
1104 
1105 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
1106   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1107   Function *ReadRegister =
1108       Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
1109   MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
1110   Value *Args[] = {MetadataAsValue::get(*C, MD)};
1111   return IRB.CreateCall(ReadRegister, Args);
1112 }
1113 
1114 bool HWAddressSanitizer::instrumentLandingPads(
1115     SmallVectorImpl<Instruction *> &LandingPadVec) {
1116   for (auto *LP : LandingPadVec) {
1117     IRBuilder<> IRB(LP->getNextNode());
1118     IRB.CreateCall(
1119         HWAsanHandleVfork,
1120         {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"
1121                                                                       : "sp")});
1122   }
1123   return true;
1124 }
1125 
1126 bool HWAddressSanitizer::instrumentStack(
1127     SmallVectorImpl<AllocaInst *> &Allocas,
1128     DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap,
1129     SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) {
1130   // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1131   // alloca addresses using that. Unfortunately, offsets are not known yet
1132   // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1133   // temp, shift-OR it into each alloca address and xor with the retag mask.
1134   // This generates one extra instruction per alloca use.
1135   for (unsigned N = 0; N < Allocas.size(); ++N) {
1136     auto *AI = Allocas[N];
1137     IRBuilder<> IRB(AI->getNextNode());
1138 
1139     // Replace uses of the alloca with tagged address.
1140     Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
1141     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
1142     Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
1143     std::string Name =
1144         AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
1145     Replacement->setName(Name + ".hwasan");
1146 
1147     AI->replaceUsesWithIf(Replacement,
1148                           [AILong](Use &U) { return U.getUser() != AILong; });
1149 
1150     for (auto *DDI : AllocaDbgMap.lookup(AI)) {
1151       // Prepend "tag_offset, N" to the dwarf expression.
1152       // Tag offset logically applies to the alloca pointer, and it makes sense
1153       // to put it at the beginning of the expression.
1154       SmallVector<uint64_t, 8> NewOps = {dwarf::DW_OP_LLVM_tag_offset,
1155                                          RetagMask(N)};
1156       DDI->setArgOperand(
1157           2, MetadataAsValue::get(*C, DIExpression::prependOpcodes(
1158                                           DDI->getExpression(), NewOps)));
1159     }
1160 
1161     size_t Size = getAllocaSizeInBytes(*AI);
1162     tagAlloca(IRB, AI, Tag, Size);
1163 
1164     for (auto RI : RetVec) {
1165       IRB.SetInsertPoint(RI);
1166 
1167       // Re-tag alloca memory with the special UAR tag.
1168       Value *Tag = getUARTag(IRB, StackTag);
1169       tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getObjectAlignment()));
1170     }
1171   }
1172 
1173   return true;
1174 }
1175 
1176 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
1177   return (AI.getAllocatedType()->isSized() &&
1178           // FIXME: instrument dynamic allocas, too
1179           AI.isStaticAlloca() &&
1180           // alloca() may be called with 0 size, ignore it.
1181           getAllocaSizeInBytes(AI) > 0 &&
1182           // We are only interested in allocas not promotable to registers.
1183           // Promotable allocas are common under -O0.
1184           !isAllocaPromotable(&AI) &&
1185           // inalloca allocas are not treated as static, and we don't want
1186           // dynamic alloca instrumentation for them as well.
1187           !AI.isUsedWithInAlloca() &&
1188           // swifterror allocas are register promoted by ISel
1189           !AI.isSwiftError());
1190 }
1191 
1192 bool HWAddressSanitizer::sanitizeFunction(Function &F) {
1193   if (&F == HwasanCtorFunction)
1194     return false;
1195 
1196   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
1197     return false;
1198 
1199   LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1200 
1201   SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1202   SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1203   SmallVector<AllocaInst*, 8> AllocasToInstrument;
1204   SmallVector<Instruction*, 8> RetVec;
1205   SmallVector<Instruction*, 8> LandingPadVec;
1206   DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> AllocaDbgMap;
1207   for (auto &BB : F) {
1208     for (auto &Inst : BB) {
1209       if (ClInstrumentStack)
1210         if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
1211           if (isInterestingAlloca(*AI))
1212             AllocasToInstrument.push_back(AI);
1213           continue;
1214         }
1215 
1216       if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
1217           isa<CleanupReturnInst>(Inst))
1218         RetVec.push_back(&Inst);
1219 
1220       if (auto *DDI = dyn_cast<DbgVariableIntrinsic>(&Inst))
1221         if (auto *Alloca =
1222                 dyn_cast_or_null<AllocaInst>(DDI->getVariableLocation()))
1223           AllocaDbgMap[Alloca].push_back(DDI);
1224 
1225       if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
1226         LandingPadVec.push_back(&Inst);
1227 
1228       getInterestingMemoryOperands(&Inst, OperandsToInstrument);
1229 
1230       if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst))
1231         IntrinToInstrument.push_back(MI);
1232     }
1233   }
1234 
1235   initializeCallbacks(*F.getParent());
1236 
1237   bool Changed = false;
1238 
1239   if (!LandingPadVec.empty())
1240     Changed |= instrumentLandingPads(LandingPadVec);
1241 
1242   if (AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1243       F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1244     // __hwasan_personality_thunk is a no-op for functions without an
1245     // instrumented stack, so we can drop it.
1246     F.setPersonalityFn(nullptr);
1247     Changed = true;
1248   }
1249 
1250   if (AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1251       IntrinToInstrument.empty())
1252     return Changed;
1253 
1254   assert(!ShadowBase);
1255 
1256   Instruction *InsertPt = &*F.getEntryBlock().begin();
1257   IRBuilder<> EntryIRB(InsertPt);
1258   emitPrologue(EntryIRB,
1259                /*WithFrameRecord*/ ClRecordStackHistory &&
1260                    !AllocasToInstrument.empty());
1261 
1262   if (!AllocasToInstrument.empty()) {
1263     Value *StackTag =
1264         ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
1265     instrumentStack(AllocasToInstrument, AllocaDbgMap, RetVec, StackTag);
1266   }
1267   // Pad and align each of the allocas that we instrumented to stop small
1268   // uninteresting allocas from hiding in instrumented alloca's padding and so
1269   // that we have enough space to store real tags for short granules.
1270   DenseMap<AllocaInst *, AllocaInst *> AllocaToPaddedAllocaMap;
1271   for (AllocaInst *AI : AllocasToInstrument) {
1272     uint64_t Size = getAllocaSizeInBytes(*AI);
1273     uint64_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
1274     AI->setAlignment(
1275         Align(std::max(AI->getAlignment(), Mapping.getObjectAlignment())));
1276     if (Size != AlignedSize) {
1277       Type *AllocatedType = AI->getAllocatedType();
1278       if (AI->isArrayAllocation()) {
1279         uint64_t ArraySize =
1280             cast<ConstantInt>(AI->getArraySize())->getZExtValue();
1281         AllocatedType = ArrayType::get(AllocatedType, ArraySize);
1282       }
1283       Type *TypeWithPadding = StructType::get(
1284           AllocatedType, ArrayType::get(Int8Ty, AlignedSize - Size));
1285       auto *NewAI = new AllocaInst(
1286           TypeWithPadding, AI->getType()->getAddressSpace(), nullptr, "", AI);
1287       NewAI->takeName(AI);
1288       NewAI->setAlignment(AI->getAlign());
1289       NewAI->setUsedWithInAlloca(AI->isUsedWithInAlloca());
1290       NewAI->setSwiftError(AI->isSwiftError());
1291       NewAI->copyMetadata(*AI);
1292       auto *Bitcast = new BitCastInst(NewAI, AI->getType(), "", AI);
1293       AI->replaceAllUsesWith(Bitcast);
1294       AllocaToPaddedAllocaMap[AI] = NewAI;
1295     }
1296   }
1297 
1298   if (!AllocaToPaddedAllocaMap.empty()) {
1299     for (auto &BB : F)
1300       for (auto &Inst : BB)
1301         if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst))
1302           if (auto *AI =
1303                   dyn_cast_or_null<AllocaInst>(DVI->getVariableLocation()))
1304             if (auto *NewAI = AllocaToPaddedAllocaMap.lookup(AI))
1305               DVI->setArgOperand(
1306                   0, MetadataAsValue::get(*C, LocalAsMetadata::get(NewAI)));
1307     for (auto &P : AllocaToPaddedAllocaMap)
1308       P.first->eraseFromParent();
1309   }
1310 
1311   // If we split the entry block, move any allocas that were originally in the
1312   // entry block back into the entry block so that they aren't treated as
1313   // dynamic allocas.
1314   if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1315     InsertPt = &*F.getEntryBlock().begin();
1316     for (auto II = EntryIRB.GetInsertBlock()->begin(),
1317               IE = EntryIRB.GetInsertBlock()->end();
1318          II != IE;) {
1319       Instruction *I = &*II++;
1320       if (auto *AI = dyn_cast<AllocaInst>(I))
1321         if (isa<ConstantInt>(AI->getArraySize()))
1322           I->moveBefore(InsertPt);
1323     }
1324   }
1325 
1326   for (auto &Operand : OperandsToInstrument)
1327     instrumentMemAccess(Operand);
1328 
1329   if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
1330     for (auto Inst : IntrinToInstrument)
1331       instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
1332   }
1333 
1334   ShadowBase = nullptr;
1335   StackBaseTag = nullptr;
1336 
1337   return true;
1338 }
1339 
1340 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1341   Constant *Initializer = GV->getInitializer();
1342   uint64_t SizeInBytes =
1343       M.getDataLayout().getTypeAllocSize(Initializer->getType());
1344   uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
1345   if (SizeInBytes != NewSize) {
1346     // Pad the initializer out to the next multiple of 16 bytes and add the
1347     // required short granule tag.
1348     std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1349     Init.back() = Tag;
1350     Constant *Padding = ConstantDataArray::get(*C, Init);
1351     Initializer = ConstantStruct::getAnon({Initializer, Padding});
1352   }
1353 
1354   auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1355                                    GlobalValue::ExternalLinkage, Initializer,
1356                                    GV->getName() + ".hwasan");
1357   NewGV->copyAttributesFrom(GV);
1358   NewGV->setLinkage(GlobalValue::PrivateLinkage);
1359   NewGV->copyMetadata(GV, 0);
1360   NewGV->setAlignment(
1361       MaybeAlign(std::max(GV->getAlignment(), Mapping.getObjectAlignment())));
1362 
1363   // It is invalid to ICF two globals that have different tags. In the case
1364   // where the size of the global is a multiple of the tag granularity the
1365   // contents of the globals may be the same but the tags (i.e. symbol values)
1366   // may be different, and the symbols are not considered during ICF. In the
1367   // case where the size is not a multiple of the granularity, the short granule
1368   // tags would discriminate two globals with different tags, but there would
1369   // otherwise be nothing stopping such a global from being incorrectly ICF'd
1370   // with an uninstrumented (i.e. tag 0) global that happened to have the short
1371   // granule tag in the last byte.
1372   NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1373 
1374   // Descriptor format (assuming little-endian):
1375   // bytes 0-3: relative address of global
1376   // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1377   // it isn't, we create multiple descriptors)
1378   // byte 7: tag
1379   auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
1380   const uint64_t MaxDescriptorSize = 0xfffff0;
1381   for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1382        DescriptorPos += MaxDescriptorSize) {
1383     auto *Descriptor =
1384         new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1385                            nullptr, GV->getName() + ".hwasan.descriptor");
1386     auto *GVRelPtr = ConstantExpr::getTrunc(
1387         ConstantExpr::getAdd(
1388             ConstantExpr::getSub(
1389                 ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1390                 ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
1391             ConstantInt::get(Int64Ty, DescriptorPos)),
1392         Int32Ty);
1393     uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
1394     auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
1395     Descriptor->setComdat(NewGV->getComdat());
1396     Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
1397     Descriptor->setSection("hwasan_globals");
1398     Descriptor->setMetadata(LLVMContext::MD_associated,
1399                             MDNode::get(*C, ValueAsMetadata::get(NewGV)));
1400     appendToCompilerUsed(M, Descriptor);
1401   }
1402 
1403   Constant *Aliasee = ConstantExpr::getIntToPtr(
1404       ConstantExpr::getAdd(
1405           ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1406           ConstantInt::get(Int64Ty, uint64_t(Tag) << kPointerTagShift)),
1407       GV->getType());
1408   auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
1409                                     GV->getLinkage(), "", Aliasee, &M);
1410   Alias->setVisibility(GV->getVisibility());
1411   Alias->takeName(GV);
1412   GV->replaceAllUsesWith(Alias);
1413   GV->eraseFromParent();
1414 }
1415 
1416 void HWAddressSanitizer::instrumentGlobals() {
1417   std::vector<GlobalVariable *> Globals;
1418   for (GlobalVariable &GV : M.globals()) {
1419     if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") ||
1420         GV.isThreadLocal())
1421       continue;
1422 
1423     // Common symbols can't have aliases point to them, so they can't be tagged.
1424     if (GV.hasCommonLinkage())
1425       continue;
1426 
1427     // Globals with custom sections may be used in __start_/__stop_ enumeration,
1428     // which would be broken both by adding tags and potentially by the extra
1429     // padding/alignment that we insert.
1430     if (GV.hasSection())
1431       continue;
1432 
1433     Globals.push_back(&GV);
1434   }
1435 
1436   MD5 Hasher;
1437   Hasher.update(M.getSourceFileName());
1438   MD5::MD5Result Hash;
1439   Hasher.final(Hash);
1440   uint8_t Tag = Hash[0];
1441 
1442   for (GlobalVariable *GV : Globals) {
1443     // Skip tag 0 in order to avoid collisions with untagged memory.
1444     if (Tag == 0)
1445       Tag = 1;
1446     instrumentGlobal(GV, Tag++);
1447   }
1448 }
1449 
1450 void HWAddressSanitizer::instrumentPersonalityFunctions() {
1451   // We need to untag stack frames as we unwind past them. That is the job of
1452   // the personality function wrapper, which either wraps an existing
1453   // personality function or acts as a personality function on its own. Each
1454   // function that has a personality function or that can be unwound past has
1455   // its personality function changed to a thunk that calls the personality
1456   // function wrapper in the runtime.
1457   MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1458   for (Function &F : M) {
1459     if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress))
1460       continue;
1461 
1462     if (F.hasPersonalityFn()) {
1463       PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F);
1464     } else if (!F.hasFnAttribute(Attribute::NoUnwind)) {
1465       PersonalityFns[nullptr].push_back(&F);
1466     }
1467   }
1468 
1469   if (PersonalityFns.empty())
1470     return;
1471 
1472   FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1473       "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty,
1474       Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy);
1475   FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy);
1476   FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy);
1477 
1478   for (auto &P : PersonalityFns) {
1479     std::string ThunkName = kHwasanPersonalityThunkName;
1480     if (P.first)
1481       ThunkName += ("." + P.first->getName()).str();
1482     FunctionType *ThunkFnTy = FunctionType::get(
1483         Int32Ty, {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int8PtrTy}, false);
1484     bool IsLocal = P.first && (!isa<GlobalValue>(P.first) ||
1485                                cast<GlobalValue>(P.first)->hasLocalLinkage());
1486     auto *ThunkFn = Function::Create(ThunkFnTy,
1487                                      IsLocal ? GlobalValue::InternalLinkage
1488                                              : GlobalValue::LinkOnceODRLinkage,
1489                                      ThunkName, &M);
1490     if (!IsLocal) {
1491       ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1492       ThunkFn->setComdat(M.getOrInsertComdat(ThunkName));
1493     }
1494 
1495     auto *BB = BasicBlock::Create(*C, "entry", ThunkFn);
1496     IRBuilder<> IRB(BB);
1497     CallInst *WrapperCall = IRB.CreateCall(
1498         HwasanPersonalityWrapper,
1499         {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2),
1500          ThunkFn->getArg(3), ThunkFn->getArg(4),
1501          P.first ? IRB.CreateBitCast(P.first, Int8PtrTy)
1502                  : Constant::getNullValue(Int8PtrTy),
1503          IRB.CreateBitCast(UnwindGetGR.getCallee(), Int8PtrTy),
1504          IRB.CreateBitCast(UnwindGetCFA.getCallee(), Int8PtrTy)});
1505     WrapperCall->setTailCall();
1506     IRB.CreateRet(WrapperCall);
1507 
1508     for (Function *F : P.second)
1509       F->setPersonalityFn(ThunkFn);
1510   }
1511 }
1512 
1513 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
1514   Scale = kDefaultShadowScale;
1515   if (ClMappingOffset.getNumOccurrences() > 0) {
1516     InGlobal = false;
1517     InTls = false;
1518     Offset = ClMappingOffset;
1519   } else if (ClEnableKhwasan || ClInstrumentWithCalls) {
1520     InGlobal = false;
1521     InTls = false;
1522     Offset = 0;
1523   } else if (ClWithIfunc) {
1524     InGlobal = true;
1525     InTls = false;
1526     Offset = kDynamicShadowSentinel;
1527   } else if (ClWithTls) {
1528     InGlobal = false;
1529     InTls = true;
1530     Offset = kDynamicShadowSentinel;
1531   } else {
1532     InGlobal = false;
1533     InTls = false;
1534     Offset = kDynamicShadowSentinel;
1535   }
1536 }
1537