1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the IRBuilder class, which is used as a convenient way
11 // to create LLVM instructions with a consistent and simplified interface.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/IR/IRBuilder.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/None.h"
18 #include "llvm/IR/Constant.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/DerivedTypes.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/GlobalValue.h"
23 #include "llvm/IR/GlobalVariable.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Intrinsics.h"
26 #include "llvm/IR/LLVMContext.h"
27 #include "llvm/IR/Operator.h"
28 #include "llvm/IR/Statepoint.h"
29 #include "llvm/IR/Type.h"
30 #include "llvm/IR/Value.h"
31 #include "llvm/Support/Casting.h"
32 #include "llvm/Support/MathExtras.h"
33 #include <cassert>
34 #include <cstdint>
35 #include <vector>
36 
37 using namespace llvm;
38 
39 /// CreateGlobalString - Make a new global variable with an initializer that
40 /// has array of i8 type filled in with the nul terminated string value
41 /// specified.  If Name is specified, it is the name of the global variable
42 /// created.
43 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
44                                                   const Twine &Name,
45                                                   unsigned AddressSpace) {
46   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
47   Module &M = *BB->getParent()->getParent();
48   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
49                                 GlobalValue::PrivateLinkage, StrConstant, Name,
50                                 nullptr, GlobalVariable::NotThreadLocal,
51                                 AddressSpace);
52   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
53   GV->setAlignment(1);
54   return GV;
55 }
56 
57 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
58   assert(BB && BB->getParent() && "No current function!");
59   return BB->getParent()->getReturnType();
60 }
61 
62 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
63   auto *PT = cast<PointerType>(Ptr->getType());
64   if (PT->getElementType()->isIntegerTy(8))
65     return Ptr;
66 
67   // Otherwise, we need to insert a bitcast.
68   PT = getInt8PtrTy(PT->getAddressSpace());
69   BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
70   BB->getInstList().insert(InsertPt, BCI);
71   SetInstDebugLocation(BCI);
72   return BCI;
73 }
74 
75 static CallInst *createCallHelper(Value *Callee, ArrayRef<Value *> Ops,
76                                   IRBuilderBase *Builder,
77                                   const Twine &Name = "",
78                                   Instruction *FMFSource = nullptr) {
79   CallInst *CI = CallInst::Create(Callee, Ops, Name);
80   if (FMFSource)
81     CI->copyFastMathFlags(FMFSource);
82   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
83   Builder->SetInstDebugLocation(CI);
84   return CI;
85 }
86 
87 static InvokeInst *createInvokeHelper(Value *Invokee, BasicBlock *NormalDest,
88                                       BasicBlock *UnwindDest,
89                                       ArrayRef<Value *> Ops,
90                                       IRBuilderBase *Builder,
91                                       const Twine &Name = "") {
92   InvokeInst *II =
93       InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
94   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
95                                                   II);
96   Builder->SetInstDebugLocation(II);
97   return II;
98 }
99 
100 CallInst *IRBuilderBase::
101 CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
102              bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
103              MDNode *NoAliasTag) {
104   Ptr = getCastedInt8PtrValue(Ptr);
105   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
106   Type *Tys[] = { Ptr->getType(), Size->getType() };
107   Module *M = BB->getParent()->getParent();
108   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
109 
110   CallInst *CI = createCallHelper(TheFn, Ops, this);
111 
112   if (Align > 0)
113     cast<MemSetInst>(CI)->setDestAlignment(Align);
114 
115   // Set the TBAA info if present.
116   if (TBAATag)
117     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
118 
119   if (ScopeTag)
120     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
121 
122   if (NoAliasTag)
123     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
124 
125   return CI;
126 }
127 
128 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
129     Value *Ptr, Value *Val, Value *Size, unsigned Align, uint32_t ElementSize,
130     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
131   assert(Align >= ElementSize &&
132          "Pointer alignment must be at least element size.");
133 
134   Ptr = getCastedInt8PtrValue(Ptr);
135   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
136   Type *Tys[] = {Ptr->getType(), Size->getType()};
137   Module *M = BB->getParent()->getParent();
138   Value *TheFn = Intrinsic::getDeclaration(
139       M, Intrinsic::memset_element_unordered_atomic, Tys);
140 
141   CallInst *CI = createCallHelper(TheFn, Ops, this);
142 
143   cast<AtomicMemSetInst>(CI)->setDestAlignment(Align);
144 
145   // Set the TBAA info if present.
146   if (TBAATag)
147     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
148 
149   if (ScopeTag)
150     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
151 
152   if (NoAliasTag)
153     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
154 
155   return CI;
156 }
157 
158 CallInst *IRBuilderBase::
159 CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
160              Value *Size, bool isVolatile, MDNode *TBAATag,
161              MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
162   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
163   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
164   Dst = getCastedInt8PtrValue(Dst);
165   Src = getCastedInt8PtrValue(Src);
166 
167   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
168   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
169   Module *M = BB->getParent()->getParent();
170   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
171 
172   CallInst *CI = createCallHelper(TheFn, Ops, this);
173 
174   auto* MCI = cast<MemCpyInst>(CI);
175   if (DstAlign > 0)
176     MCI->setDestAlignment(DstAlign);
177   if (SrcAlign > 0)
178     MCI->setSourceAlignment(SrcAlign);
179 
180   // Set the TBAA info if present.
181   if (TBAATag)
182     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
183 
184   // Set the TBAA Struct info if present.
185   if (TBAAStructTag)
186     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
187 
188   if (ScopeTag)
189     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
190 
191   if (NoAliasTag)
192     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
193 
194   return CI;
195 }
196 
197 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
198     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
199     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
200     MDNode *ScopeTag, MDNode *NoAliasTag) {
201   assert(DstAlign >= ElementSize &&
202          "Pointer alignment must be at least element size");
203   assert(SrcAlign >= ElementSize &&
204          "Pointer alignment must be at least element size");
205   Dst = getCastedInt8PtrValue(Dst);
206   Src = getCastedInt8PtrValue(Src);
207 
208   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
209   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
210   Module *M = BB->getParent()->getParent();
211   Value *TheFn = Intrinsic::getDeclaration(
212       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
213 
214   CallInst *CI = createCallHelper(TheFn, Ops, this);
215 
216   // Set the alignment of the pointer args.
217   auto *AMCI = cast<AtomicMemCpyInst>(CI);
218   AMCI->setDestAlignment(DstAlign);
219   AMCI->setSourceAlignment(SrcAlign);
220 
221   // Set the TBAA info if present.
222   if (TBAATag)
223     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
224 
225   // Set the TBAA Struct info if present.
226   if (TBAAStructTag)
227     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
228 
229   if (ScopeTag)
230     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
231 
232   if (NoAliasTag)
233     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
234 
235   return CI;
236 }
237 
238 CallInst *IRBuilderBase::
239 CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
240               Value *Size, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
241               MDNode *NoAliasTag) {
242   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
243   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
244   Dst = getCastedInt8PtrValue(Dst);
245   Src = getCastedInt8PtrValue(Src);
246 
247   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
248   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
249   Module *M = BB->getParent()->getParent();
250   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
251 
252   CallInst *CI = createCallHelper(TheFn, Ops, this);
253 
254   auto *MMI = cast<MemMoveInst>(CI);
255   if (DstAlign > 0)
256     MMI->setDestAlignment(DstAlign);
257   if (SrcAlign > 0)
258     MMI->setSourceAlignment(SrcAlign);
259 
260   // Set the TBAA info if present.
261   if (TBAATag)
262     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
263 
264   if (ScopeTag)
265     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
266 
267   if (NoAliasTag)
268     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
269 
270   return CI;
271 }
272 
273 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
274     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
275     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
276     MDNode *ScopeTag, MDNode *NoAliasTag) {
277   assert(DstAlign >= ElementSize &&
278          "Pointer alignment must be at least element size");
279   assert(SrcAlign >= ElementSize &&
280          "Pointer alignment must be at least element size");
281   Dst = getCastedInt8PtrValue(Dst);
282   Src = getCastedInt8PtrValue(Src);
283 
284   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
285   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
286   Module *M = BB->getParent()->getParent();
287   Value *TheFn = Intrinsic::getDeclaration(
288       M, Intrinsic::memmove_element_unordered_atomic, Tys);
289 
290   CallInst *CI = createCallHelper(TheFn, Ops, this);
291 
292   // Set the alignment of the pointer args.
293   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
294   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
295 
296   // Set the TBAA info if present.
297   if (TBAATag)
298     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
299 
300   // Set the TBAA Struct info if present.
301   if (TBAAStructTag)
302     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
303 
304   if (ScopeTag)
305     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
306 
307   if (NoAliasTag)
308     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
309 
310   return CI;
311 }
312 
313 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
314                                     Value *Src) {
315   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
316   Value *Ops[] = {Src};
317   Type *Tys[] = { Src->getType()->getVectorElementType(), Src->getType() };
318   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
319   return createCallHelper(Decl, Ops, Builder);
320 }
321 
322 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
323   Module *M = GetInsertBlock()->getParent()->getParent();
324   Value *Ops[] = {Acc, Src};
325   Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
326                  Src->getType()};
327   auto Decl = Intrinsic::getDeclaration(
328       M, Intrinsic::experimental_vector_reduce_fadd, Tys);
329   return createCallHelper(Decl, Ops, this);
330 }
331 
332 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
333   Module *M = GetInsertBlock()->getParent()->getParent();
334   Value *Ops[] = {Acc, Src};
335   Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
336                  Src->getType()};
337   auto Decl = Intrinsic::getDeclaration(
338       M, Intrinsic::experimental_vector_reduce_fmul, Tys);
339   return createCallHelper(Decl, Ops, this);
340 }
341 
342 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
343   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
344                                Src);
345 }
346 
347 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
348   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
349                                Src);
350 }
351 
352 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
353   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
354                                Src);
355 }
356 
357 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
358   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
359                                Src);
360 }
361 
362 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
363   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
364                                Src);
365 }
366 
367 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
368   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
369                      : Intrinsic::experimental_vector_reduce_umax;
370   return getReductionIntrinsic(this, ID, Src);
371 }
372 
373 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
374   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
375                      : Intrinsic::experimental_vector_reduce_umin;
376   return getReductionIntrinsic(this, ID, Src);
377 }
378 
379 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
380   auto Rdx = getReductionIntrinsic(
381       this, Intrinsic::experimental_vector_reduce_fmax, Src);
382   if (NoNaN) {
383     FastMathFlags FMF;
384     FMF.setNoNaNs();
385     Rdx->setFastMathFlags(FMF);
386   }
387   return Rdx;
388 }
389 
390 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
391   auto Rdx = getReductionIntrinsic(
392       this, Intrinsic::experimental_vector_reduce_fmin, Src);
393   if (NoNaN) {
394     FastMathFlags FMF;
395     FMF.setNoNaNs();
396     Rdx->setFastMathFlags(FMF);
397   }
398   return Rdx;
399 }
400 
401 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
402   assert(isa<PointerType>(Ptr->getType()) &&
403          "lifetime.start only applies to pointers.");
404   Ptr = getCastedInt8PtrValue(Ptr);
405   if (!Size)
406     Size = getInt64(-1);
407   else
408     assert(Size->getType() == getInt64Ty() &&
409            "lifetime.start requires the size to be an i64");
410   Value *Ops[] = { Size, Ptr };
411   Module *M = BB->getParent()->getParent();
412   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_start,
413                                            { Ptr->getType() });
414   return createCallHelper(TheFn, Ops, this);
415 }
416 
417 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
418   assert(isa<PointerType>(Ptr->getType()) &&
419          "lifetime.end only applies to pointers.");
420   Ptr = getCastedInt8PtrValue(Ptr);
421   if (!Size)
422     Size = getInt64(-1);
423   else
424     assert(Size->getType() == getInt64Ty() &&
425            "lifetime.end requires the size to be an i64");
426   Value *Ops[] = { Size, Ptr };
427   Module *M = BB->getParent()->getParent();
428   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_end,
429                                            { Ptr->getType() });
430   return createCallHelper(TheFn, Ops, this);
431 }
432 
433 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
434 
435   assert(isa<PointerType>(Ptr->getType()) &&
436          "invariant.start only applies to pointers.");
437   Ptr = getCastedInt8PtrValue(Ptr);
438   if (!Size)
439     Size = getInt64(-1);
440   else
441     assert(Size->getType() == getInt64Ty() &&
442            "invariant.start requires the size to be an i64");
443 
444   Value *Ops[] = {Size, Ptr};
445   // Fill in the single overloaded type: memory object type.
446   Type *ObjectPtr[1] = {Ptr->getType()};
447   Module *M = BB->getParent()->getParent();
448   Value *TheFn =
449       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
450   return createCallHelper(TheFn, Ops, this);
451 }
452 
453 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
454   assert(Cond->getType() == getInt1Ty() &&
455          "an assumption condition must be of type i1");
456 
457   Value *Ops[] = { Cond };
458   Module *M = BB->getParent()->getParent();
459   Value *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
460   return createCallHelper(FnAssume, Ops, this);
461 }
462 
463 /// Create a call to a Masked Load intrinsic.
464 /// \p Ptr      - base pointer for the load
465 /// \p Align    - alignment of the source location
466 /// \p Mask     - vector of booleans which indicates what vector lanes should
467 ///               be accessed in memory
468 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
469 ///               of the result
470 /// \p Name     - name of the result variable
471 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
472                                           Value *Mask, Value *PassThru,
473                                           const Twine &Name) {
474   auto *PtrTy = cast<PointerType>(Ptr->getType());
475   Type *DataTy = PtrTy->getElementType();
476   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
477   assert(Mask && "Mask should not be all-ones (null)");
478   if (!PassThru)
479     PassThru = UndefValue::get(DataTy);
480   Type *OverloadedTypes[] = { DataTy, PtrTy };
481   Value *Ops[] = { Ptr, getInt32(Align), Mask,  PassThru};
482   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
483                                OverloadedTypes, Name);
484 }
485 
486 /// Create a call to a Masked Store intrinsic.
487 /// \p Val   - data to be stored,
488 /// \p Ptr   - base pointer for the store
489 /// \p Align - alignment of the destination location
490 /// \p Mask  - vector of booleans which indicates what vector lanes should
491 ///            be accessed in memory
492 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
493                                            unsigned Align, Value *Mask) {
494   auto *PtrTy = cast<PointerType>(Ptr->getType());
495   Type *DataTy = PtrTy->getElementType();
496   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
497   assert(Mask && "Mask should not be all-ones (null)");
498   Type *OverloadedTypes[] = { DataTy, PtrTy };
499   Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
500   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
501 }
502 
503 /// Create a call to a Masked intrinsic, with given intrinsic Id,
504 /// an array of operands - Ops, and an array of overloaded types -
505 /// OverloadedTypes.
506 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
507                                                ArrayRef<Value *> Ops,
508                                                ArrayRef<Type *> OverloadedTypes,
509                                                const Twine &Name) {
510   Module *M = BB->getParent()->getParent();
511   Value *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
512   return createCallHelper(TheFn, Ops, this, Name);
513 }
514 
515 /// Create a call to a Masked Gather intrinsic.
516 /// \p Ptrs     - vector of pointers for loading
517 /// \p Align    - alignment for one element
518 /// \p Mask     - vector of booleans which indicates what vector lanes should
519 ///               be accessed in memory
520 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
521 ///               of the result
522 /// \p Name     - name of the result variable
523 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
524                                             Value *Mask,  Value *PassThru,
525                                             const Twine& Name) {
526   auto PtrsTy = cast<VectorType>(Ptrs->getType());
527   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
528   unsigned NumElts = PtrsTy->getVectorNumElements();
529   Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
530 
531   if (!Mask)
532     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
533                                      NumElts));
534 
535   if (!PassThru)
536     PassThru = UndefValue::get(DataTy);
537 
538   Type *OverloadedTypes[] = {DataTy, PtrsTy};
539   Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
540 
541   // We specify only one type when we create this intrinsic. Types of other
542   // arguments are derived from this type.
543   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
544                                Name);
545 }
546 
547 /// Create a call to a Masked Scatter intrinsic.
548 /// \p Data  - data to be stored,
549 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
550 ///            stored
551 /// \p Align - alignment for one element
552 /// \p Mask  - vector of booleans which indicates what vector lanes should
553 ///            be accessed in memory
554 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
555                                              unsigned Align, Value *Mask) {
556   auto PtrsTy = cast<VectorType>(Ptrs->getType());
557   auto DataTy = cast<VectorType>(Data->getType());
558   unsigned NumElts = PtrsTy->getVectorNumElements();
559 
560 #ifndef NDEBUG
561   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
562   assert(NumElts == DataTy->getVectorNumElements() &&
563          PtrTy->getElementType() == DataTy->getElementType() &&
564          "Incompatible pointer and data types");
565 #endif
566 
567   if (!Mask)
568     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
569                                      NumElts));
570 
571   Type *OverloadedTypes[] = {DataTy, PtrsTy};
572   Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
573 
574   // We specify only one type when we create this intrinsic. Types of other
575   // arguments are derived from this type.
576   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
577 }
578 
579 template <typename T0, typename T1, typename T2, typename T3>
580 static std::vector<Value *>
581 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
582                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
583                   ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
584                   ArrayRef<T3> GCArgs) {
585   std::vector<Value *> Args;
586   Args.push_back(B.getInt64(ID));
587   Args.push_back(B.getInt32(NumPatchBytes));
588   Args.push_back(ActualCallee);
589   Args.push_back(B.getInt32(CallArgs.size()));
590   Args.push_back(B.getInt32(Flags));
591   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
592   Args.push_back(B.getInt32(TransitionArgs.size()));
593   Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
594   Args.push_back(B.getInt32(DeoptArgs.size()));
595   Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
596   Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
597 
598   return Args;
599 }
600 
601 template <typename T0, typename T1, typename T2, typename T3>
602 static CallInst *CreateGCStatepointCallCommon(
603     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
604     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
605     ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
606     const Twine &Name) {
607   // Extract out the type of the callee.
608   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
609   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
610          "actual callee must be a callable value");
611 
612   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
613   // Fill in the one generic type'd argument (the function is also vararg)
614   Type *ArgTypes[] = { FuncPtrType };
615   Function *FnStatepoint =
616     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
617                               ArgTypes);
618 
619   std::vector<Value *> Args =
620       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
621                         CallArgs, TransitionArgs, DeoptArgs, GCArgs);
622   return createCallHelper(FnStatepoint, Args, Builder, Name);
623 }
624 
625 CallInst *IRBuilderBase::CreateGCStatepointCall(
626     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
627     ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
628     ArrayRef<Value *> GCArgs, const Twine &Name) {
629   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
630       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
631       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
632 }
633 
634 CallInst *IRBuilderBase::CreateGCStatepointCall(
635     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
636     ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
637     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
638   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
639       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
640       DeoptArgs, GCArgs, Name);
641 }
642 
643 CallInst *IRBuilderBase::CreateGCStatepointCall(
644     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
645     ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
646     ArrayRef<Value *> GCArgs, const Twine &Name) {
647   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
648       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
649       CallArgs, None, DeoptArgs, GCArgs, Name);
650 }
651 
652 template <typename T0, typename T1, typename T2, typename T3>
653 static InvokeInst *CreateGCStatepointInvokeCommon(
654     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
655     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
656     uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
657     ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
658   // Extract out the type of the callee.
659   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
660   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
661          "actual callee must be a callable value");
662 
663   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
664   // Fill in the one generic type'd argument (the function is also vararg)
665   Function *FnStatepoint = Intrinsic::getDeclaration(
666       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
667 
668   std::vector<Value *> Args =
669       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
670                         InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
671   return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
672                             Name);
673 }
674 
675 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
676     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
677     BasicBlock *NormalDest, BasicBlock *UnwindDest,
678     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
679     ArrayRef<Value *> GCArgs, const Twine &Name) {
680   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
681       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
682       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
683       DeoptArgs, GCArgs, Name);
684 }
685 
686 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
687     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
688     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
689     ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
690     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
691   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
692       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
693       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
694 }
695 
696 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
697     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
698     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
699     ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
700   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
701       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
702       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
703       Name);
704 }
705 
706 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
707                                        Type *ResultType,
708                                        const Twine &Name) {
709  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
710  Module *M = BB->getParent()->getParent();
711  Type *Types[] = {ResultType};
712  Value *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
713 
714  Value *Args[] = {Statepoint};
715  return createCallHelper(FnGCResult, Args, this, Name);
716 }
717 
718 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
719                                          int BaseOffset,
720                                          int DerivedOffset,
721                                          Type *ResultType,
722                                          const Twine &Name) {
723  Module *M = BB->getParent()->getParent();
724  Type *Types[] = {ResultType};
725  Value *FnGCRelocate =
726    Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
727 
728  Value *Args[] = {Statepoint,
729                   getInt32(BaseOffset),
730                   getInt32(DerivedOffset)};
731  return createCallHelper(FnGCRelocate, Args, this, Name);
732 }
733 
734 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
735                                               Instruction *FMFSource,
736                                               const Twine &Name) {
737   Module *M = BB->getModule();
738   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
739   return createCallHelper(Fn, {V}, this, Name, FMFSource);
740 }
741 
742 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
743                                                Value *RHS,
744                                                Instruction *FMFSource,
745                                                const Twine &Name) {
746   Module *M = BB->getModule();
747   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
748   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
749 }
750 
751 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
752                                          ArrayRef<Type *> Types,
753                                          ArrayRef<Value *> Args,
754                                          Instruction *FMFSource,
755                                          const Twine &Name) {
756   Module *M = BB->getModule();
757   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
758   return createCallHelper(Fn, Args, this, Name, FMFSource);
759 }
760