1 //===- MemProfiler.cpp - memory allocation and access profiler ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProfiler. Memory accesses are instrumented
10 // to increment the access count held in a shadow memory location, or
11 // alternatively to call into the runtime. Memory intrinsic calls (memmove,
12 // memcpy, memset) are changed to call the memory profiling runtime version
13 // instead.
14 //
15 //===----------------------------------------------------------------------===//
16 
17 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/ADT/Triple.h"
22 #include "llvm/Analysis/ValueTracking.h"
23 #include "llvm/IR/Constant.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/IR/Function.h"
26 #include "llvm/IR/GlobalValue.h"
27 #include "llvm/IR/IRBuilder.h"
28 #include "llvm/IR/Instruction.h"
29 #include "llvm/IR/IntrinsicInst.h"
30 #include "llvm/IR/Module.h"
31 #include "llvm/IR/Type.h"
32 #include "llvm/IR/Value.h"
33 #include "llvm/InitializePasses.h"
34 #include "llvm/Pass.h"
35 #include "llvm/ProfileData/InstrProf.h"
36 #include "llvm/Support/CommandLine.h"
37 #include "llvm/Support/Debug.h"
38 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
39 #include "llvm/Transforms/Utils/ModuleUtils.h"
40 
41 using namespace llvm;
42 
43 #define DEBUG_TYPE "memprof"
44 
45 constexpr int LLVM_MEM_PROFILER_VERSION = 1;
46 
47 // Size of memory mapped to a single shadow location.
48 constexpr uint64_t DefaultShadowGranularity = 64;
49 
50 // Scale from granularity down to shadow size.
51 constexpr uint64_t DefaultShadowScale = 3;
52 
53 constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
54 constexpr uint64_t MemProfCtorAndDtorPriority = 1;
55 // On Emscripten, the system needs more than one priorities for constructors.
56 constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
57 constexpr char MemProfInitName[] = "__memprof_init";
58 constexpr char MemProfVersionCheckNamePrefix[] =
59     "__memprof_version_mismatch_check_v";
60 
61 constexpr char MemProfShadowMemoryDynamicAddress[] =
62     "__memprof_shadow_memory_dynamic_address";
63 
64 constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
65 
66 // Command-line flags.
67 
68 static cl::opt<bool> ClInsertVersionCheck(
69     "memprof-guard-against-version-mismatch",
70     cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
71     cl::init(true));
72 
73 // This flag may need to be replaced with -f[no-]memprof-reads.
74 static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
75                                        cl::desc("instrument read instructions"),
76                                        cl::Hidden, cl::init(true));
77 
78 static cl::opt<bool>
79     ClInstrumentWrites("memprof-instrument-writes",
80                        cl::desc("instrument write instructions"), cl::Hidden,
81                        cl::init(true));
82 
83 static cl::opt<bool> ClInstrumentAtomics(
84     "memprof-instrument-atomics",
85     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
86     cl::init(true));
87 
88 static cl::opt<bool> ClUseCalls(
89     "memprof-use-callbacks",
90     cl::desc("Use callbacks instead of inline instrumentation sequences."),
91     cl::Hidden, cl::init(false));
92 
93 static cl::opt<std::string>
94     ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
95                                  cl::desc("Prefix for memory access callbacks"),
96                                  cl::Hidden, cl::init("__memprof_"));
97 
98 // These flags allow to change the shadow mapping.
99 // The shadow mapping looks like
100 //    Shadow = ((Mem & mask) >> scale) + offset
101 
102 static cl::opt<int> ClMappingScale("memprof-mapping-scale",
103                                    cl::desc("scale of memprof shadow mapping"),
104                                    cl::Hidden, cl::init(DefaultShadowScale));
105 
106 static cl::opt<int>
107     ClMappingGranularity("memprof-mapping-granularity",
108                          cl::desc("granularity of memprof shadow mapping"),
109                          cl::Hidden, cl::init(DefaultShadowGranularity));
110 
111 static cl::opt<bool> ClStack("memprof-instrument-stack",
112                              cl::desc("Instrument scalar stack variables"),
113                              cl::Hidden, cl::init(false));
114 
115 // Debug flags.
116 
117 static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
118                             cl::init(0));
119 
120 static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
121                                         cl::desc("Debug func"));
122 
123 static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
124                                cl::Hidden, cl::init(-1));
125 
126 static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
127                                cl::Hidden, cl::init(-1));
128 
129 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
130 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
131 STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
132 STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
133 
134 namespace {
135 
136 /// This struct defines the shadow mapping using the rule:
137 ///   shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
138 struct ShadowMapping {
139   ShadowMapping() {
140     Scale = ClMappingScale;
141     Granularity = ClMappingGranularity;
142     Mask = ~(Granularity - 1);
143   }
144 
145   int Scale;
146   int Granularity;
147   uint64_t Mask; // Computed as ~(Granularity-1)
148 };
149 
150 static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
151   return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
152                                        : MemProfCtorAndDtorPriority;
153 }
154 
155 struct InterestingMemoryAccess {
156   Value *Addr = nullptr;
157   bool IsWrite;
158   unsigned Alignment;
159   Type *AccessTy;
160   uint64_t TypeSize;
161   Value *MaybeMask = nullptr;
162 };
163 
164 /// Instrument the code in module to profile memory accesses.
165 class MemProfiler {
166 public:
167   MemProfiler(Module &M) {
168     C = &(M.getContext());
169     LongSize = M.getDataLayout().getPointerSizeInBits();
170     IntptrTy = Type::getIntNTy(*C, LongSize);
171   }
172 
173   /// If it is an interesting memory access, populate information
174   /// about the access and return a InterestingMemoryAccess struct.
175   /// Otherwise return None.
176   Optional<InterestingMemoryAccess>
177   isInterestingMemoryAccess(Instruction *I) const;
178 
179   void instrumentMop(Instruction *I, const DataLayout &DL,
180                      InterestingMemoryAccess &Access);
181   void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
182                          Value *Addr, uint32_t TypeSize, bool IsWrite);
183   void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
184                                    Instruction *I, Value *Addr,
185                                    unsigned Alignment, Type *AccessTy,
186                                    bool IsWrite);
187   void instrumentMemIntrinsic(MemIntrinsic *MI);
188   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
189   bool instrumentFunction(Function &F);
190   bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
191   bool insertDynamicShadowAtFunctionEntry(Function &F);
192 
193 private:
194   void initializeCallbacks(Module &M);
195 
196   LLVMContext *C;
197   int LongSize;
198   Type *IntptrTy;
199   ShadowMapping Mapping;
200 
201   // These arrays is indexed by AccessIsWrite
202   FunctionCallee MemProfMemoryAccessCallback[2];
203   FunctionCallee MemProfMemoryAccessCallbackSized[2];
204 
205   FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
206   Value *DynamicShadowOffset = nullptr;
207 };
208 
209 class MemProfilerLegacyPass : public FunctionPass {
210 public:
211   static char ID;
212 
213   explicit MemProfilerLegacyPass() : FunctionPass(ID) {
214     initializeMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
215   }
216 
217   StringRef getPassName() const override { return "MemProfilerFunctionPass"; }
218 
219   bool runOnFunction(Function &F) override {
220     MemProfiler Profiler(*F.getParent());
221     return Profiler.instrumentFunction(F);
222   }
223 };
224 
225 class ModuleMemProfiler {
226 public:
227   ModuleMemProfiler(Module &M) { TargetTriple = Triple(M.getTargetTriple()); }
228 
229   bool instrumentModule(Module &);
230 
231 private:
232   Triple TargetTriple;
233   ShadowMapping Mapping;
234   Function *MemProfCtorFunction = nullptr;
235 };
236 
237 class ModuleMemProfilerLegacyPass : public ModulePass {
238 public:
239   static char ID;
240 
241   explicit ModuleMemProfilerLegacyPass() : ModulePass(ID) {
242     initializeModuleMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
243   }
244 
245   StringRef getPassName() const override { return "ModuleMemProfiler"; }
246 
247   void getAnalysisUsage(AnalysisUsage &AU) const override {}
248 
249   bool runOnModule(Module &M) override {
250     ModuleMemProfiler MemProfiler(M);
251     return MemProfiler.instrumentModule(M);
252   }
253 };
254 
255 } // end anonymous namespace
256 
257 MemProfilerPass::MemProfilerPass() = default;
258 
259 PreservedAnalyses MemProfilerPass::run(Function &F,
260                                        AnalysisManager<Function> &AM) {
261   Module &M = *F.getParent();
262   MemProfiler Profiler(M);
263   if (Profiler.instrumentFunction(F))
264     return PreservedAnalyses::none();
265   return PreservedAnalyses::all();
266 }
267 
268 ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
269 
270 PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
271                                              AnalysisManager<Module> &AM) {
272   ModuleMemProfiler Profiler(M);
273   if (Profiler.instrumentModule(M))
274     return PreservedAnalyses::none();
275   return PreservedAnalyses::all();
276 }
277 
278 char MemProfilerLegacyPass::ID = 0;
279 
280 INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof",
281                       "MemProfiler: profile memory allocations and accesses.",
282                       false, false)
283 INITIALIZE_PASS_END(MemProfilerLegacyPass, "memprof",
284                     "MemProfiler: profile memory allocations and accesses.",
285                     false, false)
286 
287 FunctionPass *llvm::createMemProfilerFunctionPass() {
288   return new MemProfilerLegacyPass();
289 }
290 
291 char ModuleMemProfilerLegacyPass::ID = 0;
292 
293 INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module",
294                 "MemProfiler: profile memory allocations and accesses."
295                 "ModulePass",
296                 false, false)
297 
298 ModulePass *llvm::createModuleMemProfilerLegacyPassPass() {
299   return new ModuleMemProfilerLegacyPass();
300 }
301 
302 Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
303   // (Shadow & mask) >> scale
304   Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
305   Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
306   // (Shadow >> scale) | offset
307   assert(DynamicShadowOffset);
308   return IRB.CreateAdd(Shadow, DynamicShadowOffset);
309 }
310 
311 // Instrument memset/memmove/memcpy
312 void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
313   IRBuilder<> IRB(MI);
314   if (isa<MemTransferInst>(MI)) {
315     IRB.CreateCall(
316         isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
317         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
318          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
319          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
320   } else if (isa<MemSetInst>(MI)) {
321     IRB.CreateCall(
322         MemProfMemset,
323         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
324          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
325          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
326   }
327   MI->eraseFromParent();
328 }
329 
330 Optional<InterestingMemoryAccess>
331 MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
332   // Do not instrument the load fetching the dynamic shadow address.
333   if (DynamicShadowOffset == I)
334     return None;
335 
336   InterestingMemoryAccess Access;
337 
338   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
339     if (!ClInstrumentReads)
340       return None;
341     Access.IsWrite = false;
342     Access.AccessTy = LI->getType();
343     Access.Alignment = LI->getAlignment();
344     Access.Addr = LI->getPointerOperand();
345   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
346     if (!ClInstrumentWrites)
347       return None;
348     Access.IsWrite = true;
349     Access.AccessTy = SI->getValueOperand()->getType();
350     Access.Alignment = SI->getAlignment();
351     Access.Addr = SI->getPointerOperand();
352   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
353     if (!ClInstrumentAtomics)
354       return None;
355     Access.IsWrite = true;
356     Access.AccessTy = RMW->getValOperand()->getType();
357     Access.Alignment = 0;
358     Access.Addr = RMW->getPointerOperand();
359   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
360     if (!ClInstrumentAtomics)
361       return None;
362     Access.IsWrite = true;
363     Access.AccessTy = XCHG->getCompareOperand()->getType();
364     Access.Alignment = 0;
365     Access.Addr = XCHG->getPointerOperand();
366   } else if (auto *CI = dyn_cast<CallInst>(I)) {
367     auto *F = CI->getCalledFunction();
368     if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
369               F->getIntrinsicID() == Intrinsic::masked_store)) {
370       unsigned OpOffset = 0;
371       if (F->getIntrinsicID() == Intrinsic::masked_store) {
372         if (!ClInstrumentWrites)
373           return None;
374         // Masked store has an initial operand for the value.
375         OpOffset = 1;
376         Access.AccessTy = CI->getArgOperand(0)->getType();
377         Access.IsWrite = true;
378       } else {
379         if (!ClInstrumentReads)
380           return None;
381         Access.AccessTy = CI->getType();
382         Access.IsWrite = false;
383       }
384 
385       auto *BasePtr = CI->getOperand(0 + OpOffset);
386       if (auto *AlignmentConstant =
387               dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
388         Access.Alignment = (unsigned)AlignmentConstant->getZExtValue();
389       else
390         Access.Alignment = 1; // No alignment guarantees. We probably got Undef
391       Access.MaybeMask = CI->getOperand(2 + OpOffset);
392       Access.Addr = BasePtr;
393     }
394   }
395 
396   if (!Access.Addr)
397     return None;
398 
399   // Do not instrument acesses from different address spaces; we cannot deal
400   // with them.
401   Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
402   if (PtrTy->getPointerAddressSpace() != 0)
403     return None;
404 
405   // Ignore swifterror addresses.
406   // swifterror memory addresses are mem2reg promoted by instruction
407   // selection. As such they cannot have regular uses like an instrumentation
408   // function and it makes no sense to track them as memory.
409   if (Access.Addr->isSwiftError())
410     return None;
411 
412   // Peel off GEPs and BitCasts.
413   auto *Addr = Access.Addr->stripInBoundsOffsets();
414 
415   if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
416     // Do not instrument PGO counter updates.
417     if (GV->hasSection()) {
418       StringRef SectionName = GV->getSection();
419       // Check if the global is in the PGO counters section.
420       auto OF = Triple(I->getModule()->getTargetTriple()).getObjectFormat();
421       if (SectionName.endswith(
422               getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
423         return None;
424     }
425 
426     // Do not instrument accesses to LLVM internal variables.
427     if (GV->getName().startswith("__llvm"))
428       return None;
429   }
430 
431   const DataLayout &DL = I->getModule()->getDataLayout();
432   Access.TypeSize = DL.getTypeStoreSizeInBits(Access.AccessTy);
433   return Access;
434 }
435 
436 void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
437                                               Instruction *I, Value *Addr,
438                                               unsigned Alignment,
439                                               Type *AccessTy, bool IsWrite) {
440   auto *VTy = cast<FixedVectorType>(AccessTy);
441   uint64_t ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
442   unsigned Num = VTy->getNumElements();
443   auto *Zero = ConstantInt::get(IntptrTy, 0);
444   for (unsigned Idx = 0; Idx < Num; ++Idx) {
445     Value *InstrumentedAddress = nullptr;
446     Instruction *InsertBefore = I;
447     if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
448       // dyn_cast as we might get UndefValue
449       if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
450         if (Masked->isZero())
451           // Mask is constant false, so no instrumentation needed.
452           continue;
453         // If we have a true or undef value, fall through to instrumentAddress.
454         // with InsertBefore == I
455       }
456     } else {
457       IRBuilder<> IRB(I);
458       Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
459       Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
460       InsertBefore = ThenTerm;
461     }
462 
463     IRBuilder<> IRB(InsertBefore);
464     InstrumentedAddress =
465         IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
466     instrumentAddress(I, InsertBefore, InstrumentedAddress, ElemTypeSize,
467                       IsWrite);
468   }
469 }
470 
471 void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
472                                 InterestingMemoryAccess &Access) {
473   // Skip instrumentation of stack accesses unless requested.
474   if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
475     if (Access.IsWrite)
476       ++NumSkippedStackWrites;
477     else
478       ++NumSkippedStackReads;
479     return;
480   }
481 
482   if (Access.IsWrite)
483     NumInstrumentedWrites++;
484   else
485     NumInstrumentedReads++;
486 
487   if (Access.MaybeMask) {
488     instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
489                                 Access.Alignment, Access.AccessTy,
490                                 Access.IsWrite);
491   } else {
492     // Since the access counts will be accumulated across the entire allocation,
493     // we only update the shadow access count for the first location and thus
494     // don't need to worry about alignment and type size.
495     instrumentAddress(I, I, Access.Addr, Access.TypeSize, Access.IsWrite);
496   }
497 }
498 
499 void MemProfiler::instrumentAddress(Instruction *OrigIns,
500                                     Instruction *InsertBefore, Value *Addr,
501                                     uint32_t TypeSize, bool IsWrite) {
502   IRBuilder<> IRB(InsertBefore);
503   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
504 
505   if (ClUseCalls) {
506     IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
507     return;
508   }
509 
510   // Create an inline sequence to compute shadow location, and increment the
511   // value by one.
512   Type *ShadowTy = Type::getInt64Ty(*C);
513   Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
514   Value *ShadowPtr = memToShadow(AddrLong, IRB);
515   Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
516   Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
517   Value *Inc = ConstantInt::get(Type::getInt64Ty(*C), 1);
518   ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
519   IRB.CreateStore(ShadowValue, ShadowAddr);
520 }
521 
522 // Create the variable for the profile file name.
523 void createProfileFileNameVar(Module &M) {
524   const MDString *MemProfFilename =
525       dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
526   if (!MemProfFilename)
527     return;
528   assert(!MemProfFilename->getString().empty() &&
529          "Unexpected MemProfProfileFilename metadata with empty string");
530   Constant *ProfileNameConst = ConstantDataArray::getString(
531       M.getContext(), MemProfFilename->getString(), true);
532   GlobalVariable *ProfileNameVar = new GlobalVariable(
533       M, ProfileNameConst->getType(), /*isConstant=*/true,
534       GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
535   Triple TT(M.getTargetTriple());
536   if (TT.supportsCOMDAT()) {
537     ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
538     ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
539   }
540 }
541 
542 bool ModuleMemProfiler::instrumentModule(Module &M) {
543   // Create a module constructor.
544   std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
545   std::string VersionCheckName =
546       ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
547                            : "";
548   std::tie(MemProfCtorFunction, std::ignore) =
549       createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
550                                           MemProfInitName, /*InitArgTypes=*/{},
551                                           /*InitArgs=*/{}, VersionCheckName);
552 
553   const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
554   appendToGlobalCtors(M, MemProfCtorFunction, Priority);
555 
556   createProfileFileNameVar(M);
557 
558   return true;
559 }
560 
561 void MemProfiler::initializeCallbacks(Module &M) {
562   IRBuilder<> IRB(*C);
563 
564   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
565     const std::string TypeStr = AccessIsWrite ? "store" : "load";
566 
567     SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
568     SmallVector<Type *, 2> Args1{1, IntptrTy};
569     MemProfMemoryAccessCallbackSized[AccessIsWrite] =
570         M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr + "N",
571                               FunctionType::get(IRB.getVoidTy(), Args2, false));
572 
573     MemProfMemoryAccessCallback[AccessIsWrite] =
574         M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr,
575                               FunctionType::get(IRB.getVoidTy(), Args1, false));
576   }
577   MemProfMemmove = M.getOrInsertFunction(
578       ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
579       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
580   MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
581                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
582                                         IRB.getInt8PtrTy(), IntptrTy);
583   MemProfMemset = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset",
584                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
585                                         IRB.getInt32Ty(), IntptrTy);
586 }
587 
588 bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
589   // For each NSObject descendant having a +load method, this method is invoked
590   // by the ObjC runtime before any of the static constructors is called.
591   // Therefore we need to instrument such methods with a call to __memprof_init
592   // at the beginning in order to initialize our runtime before any access to
593   // the shadow memory.
594   // We cannot just ignore these methods, because they may call other
595   // instrumented functions.
596   if (F.getName().find(" load]") != std::string::npos) {
597     FunctionCallee MemProfInitFunction =
598         declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
599     IRBuilder<> IRB(&F.front(), F.front().begin());
600     IRB.CreateCall(MemProfInitFunction, {});
601     return true;
602   }
603   return false;
604 }
605 
606 bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
607   IRBuilder<> IRB(&F.front().front());
608   Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
609       MemProfShadowMemoryDynamicAddress, IntptrTy);
610   if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
611     cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
612   DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
613   return true;
614 }
615 
616 bool MemProfiler::instrumentFunction(Function &F) {
617   if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
618     return false;
619   if (ClDebugFunc == F.getName())
620     return false;
621   if (F.getName().startswith("__memprof_"))
622     return false;
623 
624   bool FunctionModified = false;
625 
626   // If needed, insert __memprof_init.
627   // This function needs to be called even if the function body is not
628   // instrumented.
629   if (maybeInsertMemProfInitAtFunctionEntry(F))
630     FunctionModified = true;
631 
632   LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
633 
634   initializeCallbacks(*F.getParent());
635 
636   SmallVector<Instruction *, 16> ToInstrument;
637 
638   // Fill the set of memory operations to instrument.
639   for (auto &BB : F) {
640     for (auto &Inst : BB) {
641       if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
642         ToInstrument.push_back(&Inst);
643     }
644   }
645 
646   if (ToInstrument.empty()) {
647     LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
648                       << " " << F << "\n");
649 
650     return FunctionModified;
651   }
652 
653   FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
654 
655   int NumInstrumented = 0;
656   for (auto *Inst : ToInstrument) {
657     if (ClDebugMin < 0 || ClDebugMax < 0 ||
658         (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
659       Optional<InterestingMemoryAccess> Access =
660           isInterestingMemoryAccess(Inst);
661       if (Access)
662         instrumentMop(Inst, F.getParent()->getDataLayout(), *Access);
663       else
664         instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
665     }
666     NumInstrumented++;
667   }
668 
669   if (NumInstrumented > 0)
670     FunctionModified = true;
671 
672   LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
673                     << F << "\n");
674 
675   return FunctionModified;
676 }
677