1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Operator.h"
27 #include "llvm/IR/NoFolder.h"
28 #include "llvm/IR/Statepoint.h"
29 #include "llvm/IR/Type.h"
30 #include "llvm/IR/Value.h"
31 #include "llvm/Support/Casting.h"
32 #include "llvm/Support/MathExtras.h"
33 #include <cassert>
34 #include <cstdint>
35 #include <vector>
36 
37 using namespace llvm;
38 
39 /// CreateGlobalString - Make a new global variable with an initializer that
40 /// has array of i8 type filled in with the nul terminated string value
41 /// specified.  If Name is specified, it is the name of the global variable
42 /// created.
43 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
44                                                   const Twine &Name,
45                                                   unsigned AddressSpace) {
46   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
47   Module &M = *BB->getParent()->getParent();
48   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
49                                 GlobalValue::PrivateLinkage, StrConstant, Name,
50                                 nullptr, GlobalVariable::NotThreadLocal,
51                                 AddressSpace);
52   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
53   GV->setAlignment(Align(1));
54   return GV;
55 }
56 
57 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
58   assert(BB && BB->getParent() && "No current function!");
59   return BB->getParent()->getReturnType();
60 }
61 
62 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
63   auto *PT = cast<PointerType>(Ptr->getType());
64   if (PT->getElementType()->isIntegerTy(8))
65     return Ptr;
66 
67   // Otherwise, we need to insert a bitcast.
68   return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
69 }
70 
71 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
72                                   IRBuilderBase *Builder,
73                                   const Twine &Name = "",
74                                   Instruction *FMFSource = nullptr) {
75   CallInst *CI = Builder->CreateCall(Callee, Ops, Name);
76   if (FMFSource)
77     CI->copyFastMathFlags(FMFSource);
78   return CI;
79 }
80 
81 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
82                                       MaybeAlign Align, bool isVolatile,
83                                       MDNode *TBAATag, MDNode *ScopeTag,
84                                       MDNode *NoAliasTag) {
85   Ptr = getCastedInt8PtrValue(Ptr);
86   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
87   Type *Tys[] = { Ptr->getType(), Size->getType() };
88   Module *M = BB->getParent()->getParent();
89   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
90 
91   CallInst *CI = createCallHelper(TheFn, Ops, this);
92 
93   if (Align)
94     cast<MemSetInst>(CI)->setDestAlignment(Align->value());
95 
96   // Set the TBAA info if present.
97   if (TBAATag)
98     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
99 
100   if (ScopeTag)
101     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
102 
103   if (NoAliasTag)
104     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
105 
106   return CI;
107 }
108 
109 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
110     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
111     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
112 
113   Ptr = getCastedInt8PtrValue(Ptr);
114   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
115   Type *Tys[] = {Ptr->getType(), Size->getType()};
116   Module *M = BB->getParent()->getParent();
117   Function *TheFn = Intrinsic::getDeclaration(
118       M, Intrinsic::memset_element_unordered_atomic, Tys);
119 
120   CallInst *CI = createCallHelper(TheFn, Ops, this);
121 
122   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
123 
124   // Set the TBAA info if present.
125   if (TBAATag)
126     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
127 
128   if (ScopeTag)
129     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
130 
131   if (NoAliasTag)
132     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
133 
134   return CI;
135 }
136 
137 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign,
138                                       Value *Src, MaybeAlign SrcAlign,
139                                       Value *Size, bool isVolatile,
140                                       MDNode *TBAATag, MDNode *TBAAStructTag,
141                                       MDNode *ScopeTag, MDNode *NoAliasTag) {
142   Dst = getCastedInt8PtrValue(Dst);
143   Src = getCastedInt8PtrValue(Src);
144 
145   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
146   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
147   Module *M = BB->getParent()->getParent();
148   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
149 
150   CallInst *CI = createCallHelper(TheFn, Ops, this);
151 
152   auto* MCI = cast<MemCpyInst>(CI);
153   if (DstAlign)
154     MCI->setDestAlignment(*DstAlign);
155   if (SrcAlign)
156     MCI->setSourceAlignment(*SrcAlign);
157 
158   // Set the TBAA info if present.
159   if (TBAATag)
160     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
161 
162   // Set the TBAA Struct info if present.
163   if (TBAAStructTag)
164     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
165 
166   if (ScopeTag)
167     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
168 
169   if (NoAliasTag)
170     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
171 
172   return CI;
173 }
174 
175 CallInst *IRBuilderBase::CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign,
176                                             Value *Src, MaybeAlign SrcAlign,
177                                             Value *Size) {
178   Dst = getCastedInt8PtrValue(Dst);
179   Src = getCastedInt8PtrValue(Src);
180   Value *IsVolatile = getInt1(false);
181 
182   Value *Ops[] = {Dst, Src, Size, IsVolatile};
183   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
184   Function *F = BB->getParent();
185   Module *M = F->getParent();
186   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
187 
188   CallInst *CI = createCallHelper(TheFn, Ops, this);
189 
190   auto *MCI = cast<MemCpyInlineInst>(CI);
191   if (DstAlign)
192     MCI->setDestAlignment(*DstAlign);
193   if (SrcAlign)
194     MCI->setSourceAlignment(*SrcAlign);
195 
196   return CI;
197 }
198 
199 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
200     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
201     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
202     MDNode *ScopeTag, MDNode *NoAliasTag) {
203   assert(DstAlign >= ElementSize &&
204          "Pointer alignment must be at least element size");
205   assert(SrcAlign >= ElementSize &&
206          "Pointer alignment must be at least element size");
207   Dst = getCastedInt8PtrValue(Dst);
208   Src = getCastedInt8PtrValue(Src);
209 
210   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
211   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
212   Module *M = BB->getParent()->getParent();
213   Function *TheFn = Intrinsic::getDeclaration(
214       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
215 
216   CallInst *CI = createCallHelper(TheFn, Ops, this);
217 
218   // Set the alignment of the pointer args.
219   auto *AMCI = cast<AtomicMemCpyInst>(CI);
220   AMCI->setDestAlignment(DstAlign);
221   AMCI->setSourceAlignment(SrcAlign);
222 
223   // Set the TBAA info if present.
224   if (TBAATag)
225     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
226 
227   // Set the TBAA Struct info if present.
228   if (TBAAStructTag)
229     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
230 
231   if (ScopeTag)
232     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
233 
234   if (NoAliasTag)
235     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
236 
237   return CI;
238 }
239 
240 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
241                                        Value *Src, MaybeAlign SrcAlign,
242                                        Value *Size, bool isVolatile,
243                                        MDNode *TBAATag, MDNode *ScopeTag,
244                                        MDNode *NoAliasTag) {
245   Dst = getCastedInt8PtrValue(Dst);
246   Src = getCastedInt8PtrValue(Src);
247 
248   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
249   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
250   Module *M = BB->getParent()->getParent();
251   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
252 
253   CallInst *CI = createCallHelper(TheFn, Ops, this);
254 
255   auto *MMI = cast<MemMoveInst>(CI);
256   if (DstAlign)
257     MMI->setDestAlignment(*DstAlign);
258   if (SrcAlign)
259     MMI->setSourceAlignment(*SrcAlign);
260 
261   // Set the TBAA info if present.
262   if (TBAATag)
263     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
264 
265   if (ScopeTag)
266     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
267 
268   if (NoAliasTag)
269     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
270 
271   return CI;
272 }
273 
274 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
275     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
276     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
277     MDNode *ScopeTag, MDNode *NoAliasTag) {
278   assert(DstAlign >= ElementSize &&
279          "Pointer alignment must be at least element size");
280   assert(SrcAlign >= ElementSize &&
281          "Pointer alignment must be at least element size");
282   Dst = getCastedInt8PtrValue(Dst);
283   Src = getCastedInt8PtrValue(Src);
284 
285   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
286   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
287   Module *M = BB->getParent()->getParent();
288   Function *TheFn = Intrinsic::getDeclaration(
289       M, Intrinsic::memmove_element_unordered_atomic, Tys);
290 
291   CallInst *CI = createCallHelper(TheFn, Ops, this);
292 
293   // Set the alignment of the pointer args.
294   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
295   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
296 
297   // Set the TBAA info if present.
298   if (TBAATag)
299     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
300 
301   // Set the TBAA Struct info if present.
302   if (TBAAStructTag)
303     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
304 
305   if (ScopeTag)
306     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
307 
308   if (NoAliasTag)
309     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
310 
311   return CI;
312 }
313 
314 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
315                                     Value *Src) {
316   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
317   Value *Ops[] = {Src};
318   Type *Tys[] = { Src->getType() };
319   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
320   return createCallHelper(Decl, Ops, Builder);
321 }
322 
323 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
324   Module *M = GetInsertBlock()->getParent()->getParent();
325   Value *Ops[] = {Acc, Src};
326   Type *Tys[] = {Acc->getType(), Src->getType()};
327   auto Decl = Intrinsic::getDeclaration(
328       M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
329   return createCallHelper(Decl, Ops, this);
330 }
331 
332 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
333   Module *M = GetInsertBlock()->getParent()->getParent();
334   Value *Ops[] = {Acc, Src};
335   Type *Tys[] = {Acc->getType(), Src->getType()};
336   auto Decl = Intrinsic::getDeclaration(
337       M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
338   return createCallHelper(Decl, Ops, this);
339 }
340 
341 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
342   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
343                                Src);
344 }
345 
346 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
347   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
348                                Src);
349 }
350 
351 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
352   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
353                                Src);
354 }
355 
356 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
357   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
358                                Src);
359 }
360 
361 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
362   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
363                                Src);
364 }
365 
366 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
367   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
368                      : Intrinsic::experimental_vector_reduce_umax;
369   return getReductionIntrinsic(this, ID, Src);
370 }
371 
372 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
373   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
374                      : Intrinsic::experimental_vector_reduce_umin;
375   return getReductionIntrinsic(this, ID, Src);
376 }
377 
378 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
379   auto Rdx = getReductionIntrinsic(
380       this, Intrinsic::experimental_vector_reduce_fmax, Src);
381   if (NoNaN) {
382     FastMathFlags FMF;
383     FMF.setNoNaNs();
384     Rdx->setFastMathFlags(FMF);
385   }
386   return Rdx;
387 }
388 
389 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
390   auto Rdx = getReductionIntrinsic(
391       this, Intrinsic::experimental_vector_reduce_fmin, Src);
392   if (NoNaN) {
393     FastMathFlags FMF;
394     FMF.setNoNaNs();
395     Rdx->setFastMathFlags(FMF);
396   }
397   return Rdx;
398 }
399 
400 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
401   assert(isa<PointerType>(Ptr->getType()) &&
402          "lifetime.start only applies to pointers.");
403   Ptr = getCastedInt8PtrValue(Ptr);
404   if (!Size)
405     Size = getInt64(-1);
406   else
407     assert(Size->getType() == getInt64Ty() &&
408            "lifetime.start requires the size to be an i64");
409   Value *Ops[] = { Size, Ptr };
410   Module *M = BB->getParent()->getParent();
411   Function *TheFn =
412       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
413   return createCallHelper(TheFn, Ops, this);
414 }
415 
416 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
417   assert(isa<PointerType>(Ptr->getType()) &&
418          "lifetime.end only applies to pointers.");
419   Ptr = getCastedInt8PtrValue(Ptr);
420   if (!Size)
421     Size = getInt64(-1);
422   else
423     assert(Size->getType() == getInt64Ty() &&
424            "lifetime.end requires the size to be an i64");
425   Value *Ops[] = { Size, Ptr };
426   Module *M = BB->getParent()->getParent();
427   Function *TheFn =
428       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
429   return createCallHelper(TheFn, Ops, this);
430 }
431 
432 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
433 
434   assert(isa<PointerType>(Ptr->getType()) &&
435          "invariant.start only applies to pointers.");
436   Ptr = getCastedInt8PtrValue(Ptr);
437   if (!Size)
438     Size = getInt64(-1);
439   else
440     assert(Size->getType() == getInt64Ty() &&
441            "invariant.start requires the size to be an i64");
442 
443   Value *Ops[] = {Size, Ptr};
444   // Fill in the single overloaded type: memory object type.
445   Type *ObjectPtr[1] = {Ptr->getType()};
446   Module *M = BB->getParent()->getParent();
447   Function *TheFn =
448       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
449   return createCallHelper(TheFn, Ops, this);
450 }
451 
452 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
453   assert(Cond->getType() == getInt1Ty() &&
454          "an assumption condition must be of type i1");
455 
456   Value *Ops[] = { Cond };
457   Module *M = BB->getParent()->getParent();
458   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
459   return createCallHelper(FnAssume, Ops, this);
460 }
461 
462 /// Create a call to a Masked Load intrinsic.
463 /// \p Ptr       - base pointer for the load
464 /// \p Alignment - alignment of the source location
465 /// \p Mask      - vector of booleans which indicates what vector lanes should
466 ///                be accessed in memory
467 /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
468 ///                of the result
469 /// \p Name      - name of the result variable
470 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment,
471                                           Value *Mask, Value *PassThru,
472                                           const Twine &Name) {
473   auto *PtrTy = cast<PointerType>(Ptr->getType());
474   Type *DataTy = PtrTy->getElementType();
475   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
476   assert(Mask && "Mask should not be all-ones (null)");
477   if (!PassThru)
478     PassThru = UndefValue::get(DataTy);
479   Type *OverloadedTypes[] = { DataTy, PtrTy };
480   Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
481   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
482                                OverloadedTypes, Name);
483 }
484 
485 /// Create a call to a Masked Store intrinsic.
486 /// \p Val       - data to be stored,
487 /// \p Ptr       - base pointer for the store
488 /// \p Alignment - alignment of the destination location
489 /// \p Mask      - vector of booleans which indicates what vector lanes should
490 ///                be accessed in memory
491 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
492                                            Align Alignment, Value *Mask) {
493   auto *PtrTy = cast<PointerType>(Ptr->getType());
494   Type *DataTy = PtrTy->getElementType();
495   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
496   assert(Mask && "Mask should not be all-ones (null)");
497   Type *OverloadedTypes[] = { DataTy, PtrTy };
498   Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
499   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
500 }
501 
502 /// Create a call to a Masked intrinsic, with given intrinsic Id,
503 /// an array of operands - Ops, and an array of overloaded types -
504 /// OverloadedTypes.
505 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
506                                                ArrayRef<Value *> Ops,
507                                                ArrayRef<Type *> OverloadedTypes,
508                                                const Twine &Name) {
509   Module *M = BB->getParent()->getParent();
510   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
511   return createCallHelper(TheFn, Ops, this, Name);
512 }
513 
514 /// Create a call to a Masked Gather intrinsic.
515 /// \p Ptrs     - vector of pointers for loading
516 /// \p Align    - alignment for one element
517 /// \p Mask     - vector of booleans which indicates what vector lanes should
518 ///               be accessed in memory
519 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
520 ///               of the result
521 /// \p Name     - name of the result variable
522 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
523                                             Value *Mask, Value *PassThru,
524                                             const Twine &Name) {
525   auto PtrsTy = cast<VectorType>(Ptrs->getType());
526   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
527   unsigned NumElts = PtrsTy->getNumElements();
528   Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
529 
530   if (!Mask)
531     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
532                                      NumElts));
533 
534   if (!PassThru)
535     PassThru = UndefValue::get(DataTy);
536 
537   Type *OverloadedTypes[] = {DataTy, PtrsTy};
538   Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
539 
540   // We specify only one type when we create this intrinsic. Types of other
541   // arguments are derived from this type.
542   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
543                                Name);
544 }
545 
546 /// Create a call to a Masked Scatter intrinsic.
547 /// \p Data  - data to be stored,
548 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
549 ///            stored
550 /// \p Align - alignment for one element
551 /// \p Mask  - vector of booleans which indicates what vector lanes should
552 ///            be accessed in memory
553 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
554                                              Align Alignment, Value *Mask) {
555   auto PtrsTy = cast<VectorType>(Ptrs->getType());
556   auto DataTy = cast<VectorType>(Data->getType());
557   unsigned NumElts = PtrsTy->getNumElements();
558 
559 #ifndef NDEBUG
560   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
561   assert(NumElts == DataTy->getNumElements() &&
562          PtrTy->getElementType() == DataTy->getElementType() &&
563          "Incompatible pointer and data types");
564 #endif
565 
566   if (!Mask)
567     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
568                                      NumElts));
569 
570   Type *OverloadedTypes[] = {DataTy, PtrsTy};
571   Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
572 
573   // We specify only one type when we create this intrinsic. Types of other
574   // arguments are derived from this type.
575   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
576 }
577 
578 template <typename T0, typename T1, typename T2, typename T3>
579 static std::vector<Value *>
580 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
581                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
582                   ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
583                   ArrayRef<T3> GCArgs) {
584   std::vector<Value *> Args;
585   Args.push_back(B.getInt64(ID));
586   Args.push_back(B.getInt32(NumPatchBytes));
587   Args.push_back(ActualCallee);
588   Args.push_back(B.getInt32(CallArgs.size()));
589   Args.push_back(B.getInt32(Flags));
590   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
591   Args.push_back(B.getInt32(TransitionArgs.size()));
592   Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
593   Args.push_back(B.getInt32(DeoptArgs.size()));
594   Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
595   Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
596 
597   return Args;
598 }
599 
600 template <typename T0, typename T1, typename T2, typename T3>
601 static CallInst *CreateGCStatepointCallCommon(
602     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
603     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
604     ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
605     const Twine &Name) {
606   // Extract out the type of the callee.
607   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
608   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
609          "actual callee must be a callable value");
610 
611   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
612   // Fill in the one generic type'd argument (the function is also vararg)
613   Type *ArgTypes[] = { FuncPtrType };
614   Function *FnStatepoint =
615     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
616                               ArgTypes);
617 
618   std::vector<Value *> Args =
619       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
620                         CallArgs, TransitionArgs, DeoptArgs, GCArgs);
621   return createCallHelper(FnStatepoint, Args, Builder, Name);
622 }
623 
624 CallInst *IRBuilderBase::CreateGCStatepointCall(
625     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
626     ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
627     ArrayRef<Value *> GCArgs, const Twine &Name) {
628   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
629       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
630       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
631 }
632 
633 CallInst *IRBuilderBase::CreateGCStatepointCall(
634     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
635     ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
636     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
637   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
638       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
639       DeoptArgs, GCArgs, Name);
640 }
641 
642 CallInst *IRBuilderBase::CreateGCStatepointCall(
643     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
644     ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
645     ArrayRef<Value *> GCArgs, const Twine &Name) {
646   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
647       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
648       CallArgs, None, DeoptArgs, GCArgs, Name);
649 }
650 
651 template <typename T0, typename T1, typename T2, typename T3>
652 static InvokeInst *CreateGCStatepointInvokeCommon(
653     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
654     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
655     uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
656     ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
657   // Extract out the type of the callee.
658   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
659   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
660          "actual callee must be a callable value");
661 
662   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
663   // Fill in the one generic type'd argument (the function is also vararg)
664   Function *FnStatepoint = Intrinsic::getDeclaration(
665       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
666 
667   std::vector<Value *> Args =
668       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
669                         InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
670   return Builder->CreateInvoke(FnStatepoint, NormalDest, UnwindDest, Args,
671                                Name);
672 }
673 
674 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
675     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
676     BasicBlock *NormalDest, BasicBlock *UnwindDest,
677     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
678     ArrayRef<Value *> GCArgs, const Twine &Name) {
679   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
680       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
681       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
682       DeoptArgs, GCArgs, Name);
683 }
684 
685 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
686     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
687     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
688     ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
689     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
690   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
691       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
692       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
693 }
694 
695 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
696     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
697     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
698     ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
699   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
700       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
701       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
702       Name);
703 }
704 
705 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
706                                        Type *ResultType,
707                                        const Twine &Name) {
708  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
709  Module *M = BB->getParent()->getParent();
710  Type *Types[] = {ResultType};
711  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
712 
713  Value *Args[] = {Statepoint};
714  return createCallHelper(FnGCResult, Args, this, Name);
715 }
716 
717 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
718                                          int BaseOffset,
719                                          int DerivedOffset,
720                                          Type *ResultType,
721                                          const Twine &Name) {
722  Module *M = BB->getParent()->getParent();
723  Type *Types[] = {ResultType};
724  Function *FnGCRelocate =
725      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
726 
727  Value *Args[] = {Statepoint,
728                   getInt32(BaseOffset),
729                   getInt32(DerivedOffset)};
730  return createCallHelper(FnGCRelocate, Args, this, Name);
731 }
732 
733 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
734                                               Instruction *FMFSource,
735                                               const Twine &Name) {
736   Module *M = BB->getModule();
737   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
738   return createCallHelper(Fn, {V}, this, Name, FMFSource);
739 }
740 
741 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
742                                                Value *RHS,
743                                                Instruction *FMFSource,
744                                                const Twine &Name) {
745   Module *M = BB->getModule();
746   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
747   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
748 }
749 
750 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
751                                          ArrayRef<Type *> Types,
752                                          ArrayRef<Value *> Args,
753                                          Instruction *FMFSource,
754                                          const Twine &Name) {
755   Module *M = BB->getModule();
756   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
757   return createCallHelper(Fn, Args, this, Name, FMFSource);
758 }
759 
760 CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
761     Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
762     const Twine &Name, MDNode *FPMathTag,
763     Optional<RoundingMode> Rounding,
764     Optional<fp::ExceptionBehavior> Except) {
765   Value *RoundingV = getConstrainedFPRounding(Rounding);
766   Value *ExceptV = getConstrainedFPExcept(Except);
767 
768   FastMathFlags UseFMF = FMF;
769   if (FMFSource)
770     UseFMF = FMFSource->getFastMathFlags();
771 
772   CallInst *C = CreateIntrinsic(ID, {L->getType()},
773                                 {L, R, RoundingV, ExceptV}, nullptr, Name);
774   setConstrainedFPCallAttr(C);
775   setFPAttrs(C, FPMathTag, UseFMF);
776   return C;
777 }
778 
779 Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
780                                    const Twine &Name, MDNode *FPMathTag) {
781   if (Instruction::isBinaryOp(Opc)) {
782     assert(Ops.size() == 2 && "Invalid number of operands!");
783     return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
784                        Ops[0], Ops[1], Name, FPMathTag);
785   }
786   if (Instruction::isUnaryOp(Opc)) {
787     assert(Ops.size() == 1 && "Invalid number of operands!");
788     return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
789                       Ops[0], Name, FPMathTag);
790   }
791   llvm_unreachable("Unexpected opcode!");
792 }
793 
794 CallInst *IRBuilderBase::CreateConstrainedFPCast(
795     Intrinsic::ID ID, Value *V, Type *DestTy,
796     Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
797     Optional<RoundingMode> Rounding,
798     Optional<fp::ExceptionBehavior> Except) {
799   Value *ExceptV = getConstrainedFPExcept(Except);
800 
801   FastMathFlags UseFMF = FMF;
802   if (FMFSource)
803     UseFMF = FMFSource->getFastMathFlags();
804 
805   CallInst *C;
806   bool HasRoundingMD = false;
807   switch (ID) {
808   default:
809     break;
810 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
811   case Intrinsic::INTRINSIC:                                \
812     HasRoundingMD = ROUND_MODE;                             \
813     break;
814 #include "llvm/IR/ConstrainedOps.def"
815   }
816   if (HasRoundingMD) {
817     Value *RoundingV = getConstrainedFPRounding(Rounding);
818     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
819                         nullptr, Name);
820   } else
821     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
822                         Name);
823 
824   setConstrainedFPCallAttr(C);
825 
826   if (isa<FPMathOperator>(C))
827     setFPAttrs(C, FPMathTag, UseFMF);
828   return C;
829 }
830 
831 Value *IRBuilderBase::CreateFCmpHelper(
832     CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
833     MDNode *FPMathTag, bool IsSignaling) {
834   if (IsFPConstrained) {
835     auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
836                           : Intrinsic::experimental_constrained_fcmp;
837     return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
838   }
839 
840   if (auto *LC = dyn_cast<Constant>(LHS))
841     if (auto *RC = dyn_cast<Constant>(RHS))
842       return Insert(Folder.CreateFCmp(P, LC, RC), Name);
843   return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
844 }
845 
846 CallInst *IRBuilderBase::CreateConstrainedFPCmp(
847     Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
848     const Twine &Name, Optional<fp::ExceptionBehavior> Except) {
849   Value *PredicateV = getConstrainedFPPredicate(P);
850   Value *ExceptV = getConstrainedFPExcept(Except);
851 
852   CallInst *C = CreateIntrinsic(ID, {L->getType()},
853                                 {L, R, PredicateV, ExceptV}, nullptr, Name);
854   setConstrainedFPCallAttr(C);
855   return C;
856 }
857 
858 CallInst *IRBuilderBase::CreateConstrainedFPCall(
859     Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
860     Optional<RoundingMode> Rounding,
861     Optional<fp::ExceptionBehavior> Except) {
862   llvm::SmallVector<Value *, 6> UseArgs;
863 
864   for (auto *OneArg : Args)
865     UseArgs.push_back(OneArg);
866   bool HasRoundingMD = false;
867   switch (Callee->getIntrinsicID()) {
868   default:
869     break;
870 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
871   case Intrinsic::INTRINSIC:                                \
872     HasRoundingMD = ROUND_MODE;                             \
873     break;
874 #include "llvm/IR/ConstrainedOps.def"
875   }
876   if (HasRoundingMD)
877     UseArgs.push_back(getConstrainedFPRounding(Rounding));
878   UseArgs.push_back(getConstrainedFPExcept(Except));
879 
880   CallInst *C = CreateCall(Callee, UseArgs, Name);
881   setConstrainedFPCallAttr(C);
882   return C;
883 }
884 
885 Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
886                                    const Twine &Name, Instruction *MDFrom) {
887   if (auto *CC = dyn_cast<Constant>(C))
888     if (auto *TC = dyn_cast<Constant>(True))
889       if (auto *FC = dyn_cast<Constant>(False))
890         return Insert(Folder.CreateSelect(CC, TC, FC), Name);
891 
892   SelectInst *Sel = SelectInst::Create(C, True, False);
893   if (MDFrom) {
894     MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
895     MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
896     Sel = addBranchMetadata(Sel, Prof, Unpred);
897   }
898   if (isa<FPMathOperator>(Sel))
899     setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
900   return Insert(Sel, Name);
901 }
902 
903 Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS,
904                                     const Twine &Name) {
905   assert(LHS->getType() == RHS->getType() &&
906          "Pointer subtraction operand types must match!");
907   auto *ArgType = cast<PointerType>(LHS->getType());
908   Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));
909   Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));
910   Value *Difference = CreateSub(LHS_int, RHS_int);
911   return CreateExactSDiv(Difference,
912                          ConstantExpr::getSizeOf(ArgType->getElementType()),
913                          Name);
914 }
915 
916 Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
917   assert(isa<PointerType>(Ptr->getType()) &&
918          "launder.invariant.group only applies to pointers.");
919   // FIXME: we could potentially avoid casts to/from i8*.
920   auto *PtrType = Ptr->getType();
921   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
922   if (PtrType != Int8PtrTy)
923     Ptr = CreateBitCast(Ptr, Int8PtrTy);
924   Module *M = BB->getParent()->getParent();
925   Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
926       M, Intrinsic::launder_invariant_group, {Int8PtrTy});
927 
928   assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
929          FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
930              Int8PtrTy &&
931          "LaunderInvariantGroup should take and return the same type");
932 
933   CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
934 
935   if (PtrType != Int8PtrTy)
936     return CreateBitCast(Fn, PtrType);
937   return Fn;
938 }
939 
940 Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
941   assert(isa<PointerType>(Ptr->getType()) &&
942          "strip.invariant.group only applies to pointers.");
943 
944   // FIXME: we could potentially avoid casts to/from i8*.
945   auto *PtrType = Ptr->getType();
946   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
947   if (PtrType != Int8PtrTy)
948     Ptr = CreateBitCast(Ptr, Int8PtrTy);
949   Module *M = BB->getParent()->getParent();
950   Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
951       M, Intrinsic::strip_invariant_group, {Int8PtrTy});
952 
953   assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
954          FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
955              Int8PtrTy &&
956          "StripInvariantGroup should take and return the same type");
957 
958   CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
959 
960   if (PtrType != Int8PtrTy)
961     return CreateBitCast(Fn, PtrType);
962   return Fn;
963 }
964 
965 Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
966                                         const Twine &Name) {
967   assert(NumElts > 0 && "Cannot splat to an empty vector!");
968 
969   // First insert it into an undef vector so we can shuffle it.
970   Type *I32Ty = getInt32Ty();
971   Value *Undef = UndefValue::get(VectorType::get(V->getType(), NumElts));
972   V = CreateInsertElement(Undef, V, ConstantInt::get(I32Ty, 0),
973                           Name + ".splatinsert");
974 
975   // Shuffle the value across the desired number of elements.
976   Value *Zeros = ConstantAggregateZero::get(VectorType::get(I32Ty, NumElts));
977   return CreateShuffleVector(V, Undef, Zeros, Name + ".splat");
978 }
979 
980 Value *IRBuilderBase::CreateExtractInteger(
981     const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
982     uint64_t Offset, const Twine &Name) {
983   auto *IntTy = cast<IntegerType>(From->getType());
984   assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
985              DL.getTypeStoreSize(IntTy) &&
986          "Element extends past full value");
987   uint64_t ShAmt = 8 * Offset;
988   Value *V = From;
989   if (DL.isBigEndian())
990     ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
991                  DL.getTypeStoreSize(ExtractedTy) - Offset);
992   if (ShAmt) {
993     V = CreateLShr(V, ShAmt, Name + ".shift");
994   }
995   assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
996          "Cannot extract to a larger integer!");
997   if (ExtractedTy != IntTy) {
998     V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
999   }
1000   return V;
1001 }
1002 
1003 Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
1004     Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1005     MDNode *DbgInfo) {
1006   assert(isa<PointerType>(Base->getType()) &&
1007          "Invalid Base ptr type for preserve.array.access.index.");
1008   auto *BaseType = Base->getType();
1009 
1010   Value *LastIndexV = getInt32(LastIndex);
1011   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1012   SmallVector<Value *, 4> IdxList;
1013   for (unsigned I = 0; I < Dimension; ++I)
1014     IdxList.push_back(Zero);
1015   IdxList.push_back(LastIndexV);
1016 
1017   Type *ResultType =
1018       GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList);
1019 
1020   Module *M = BB->getParent()->getParent();
1021   Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1022       M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1023 
1024   Value *DimV = getInt32(Dimension);
1025   CallInst *Fn =
1026       CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1027   if (DbgInfo)
1028     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1029 
1030   return Fn;
1031 }
1032 
1033 Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1034     Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1035   assert(isa<PointerType>(Base->getType()) &&
1036          "Invalid Base ptr type for preserve.union.access.index.");
1037   auto *BaseType = Base->getType();
1038 
1039   Module *M = BB->getParent()->getParent();
1040   Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1041       M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1042 
1043   Value *DIIndex = getInt32(FieldIndex);
1044   CallInst *Fn =
1045       CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1046   if (DbgInfo)
1047     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1048 
1049   return Fn;
1050 }
1051 
1052 Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1053     Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1054     MDNode *DbgInfo) {
1055   assert(isa<PointerType>(Base->getType()) &&
1056          "Invalid Base ptr type for preserve.struct.access.index.");
1057   auto *BaseType = Base->getType();
1058 
1059   Value *GEPIndex = getInt32(Index);
1060   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1061   Type *ResultType =
1062       GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1063 
1064   Module *M = BB->getParent()->getParent();
1065   Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1066       M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1067 
1068   Value *DIIndex = getInt32(FieldIndex);
1069   CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1070                             {Base, GEPIndex, DIIndex});
1071   if (DbgInfo)
1072     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1073 
1074   return Fn;
1075 }
1076 
1077 CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(
1078     const DataLayout &DL, Value *PtrValue, Value *Mask, Type *IntPtrTy,
1079     Value *OffsetValue, Value **TheCheck) {
1080   Value *PtrIntValue = CreatePtrToInt(PtrValue, IntPtrTy, "ptrint");
1081 
1082   if (OffsetValue) {
1083     bool IsOffsetZero = false;
1084     if (const auto *CI = dyn_cast<ConstantInt>(OffsetValue))
1085       IsOffsetZero = CI->isZero();
1086 
1087     if (!IsOffsetZero) {
1088       if (OffsetValue->getType() != IntPtrTy)
1089         OffsetValue = CreateIntCast(OffsetValue, IntPtrTy, /*isSigned*/ true,
1090                                     "offsetcast");
1091       PtrIntValue = CreateSub(PtrIntValue, OffsetValue, "offsetptr");
1092     }
1093   }
1094 
1095   Value *Zero = ConstantInt::get(IntPtrTy, 0);
1096   Value *MaskedPtr = CreateAnd(PtrIntValue, Mask, "maskedptr");
1097   Value *InvCond = CreateICmpEQ(MaskedPtr, Zero, "maskcond");
1098   if (TheCheck)
1099     *TheCheck = InvCond;
1100 
1101   return CreateAssumption(InvCond);
1102 }
1103 
1104 CallInst *IRBuilderBase::CreateAlignmentAssumption(
1105     const DataLayout &DL, Value *PtrValue, unsigned Alignment,
1106     Value *OffsetValue, Value **TheCheck) {
1107   assert(isa<PointerType>(PtrValue->getType()) &&
1108          "trying to create an alignment assumption on a non-pointer?");
1109   assert(Alignment != 0 && "Invalid Alignment");
1110   auto *PtrTy = cast<PointerType>(PtrValue->getType());
1111   Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1112 
1113   Value *Mask = ConstantInt::get(IntPtrTy, Alignment - 1);
1114   return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy,
1115                                          OffsetValue, TheCheck);
1116 }
1117 
1118 CallInst *IRBuilderBase::CreateAlignmentAssumption(
1119     const DataLayout &DL, Value *PtrValue, Value *Alignment,
1120     Value *OffsetValue, Value **TheCheck) {
1121   assert(isa<PointerType>(PtrValue->getType()) &&
1122          "trying to create an alignment assumption on a non-pointer?");
1123   auto *PtrTy = cast<PointerType>(PtrValue->getType());
1124   Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1125 
1126   if (Alignment->getType() != IntPtrTy)
1127     Alignment = CreateIntCast(Alignment, IntPtrTy, /*isSigned*/ false,
1128                               "alignmentcast");
1129 
1130   Value *Mask = CreateSub(Alignment, ConstantInt::get(IntPtrTy, 1), "mask");
1131 
1132   return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy,
1133                                          OffsetValue, TheCheck);
1134 }
1135 
1136 IRBuilderDefaultInserter::~IRBuilderDefaultInserter() {}
1137 IRBuilderCallbackInserter::~IRBuilderCallbackInserter() {}
1138 IRBuilderFolder::~IRBuilderFolder() {}
1139 void ConstantFolder::anchor() {}
1140 void NoFolder::anchor() {}
1141