1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Operator.h"
27 #include "llvm/IR/Statepoint.h"
28 #include "llvm/IR/Type.h"
29 #include "llvm/IR/Value.h"
30 #include "llvm/Support/Casting.h"
31 #include "llvm/Support/MathExtras.h"
32 #include <cassert>
33 #include <cstdint>
34 #include <vector>
35 
36 using namespace llvm;
37 
38 /// CreateGlobalString - Make a new global variable with an initializer that
39 /// has array of i8 type filled in with the nul terminated string value
40 /// specified.  If Name is specified, it is the name of the global variable
41 /// created.
42 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
43                                                   const Twine &Name,
44                                                   unsigned AddressSpace) {
45   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
46   Module &M = *BB->getParent()->getParent();
47   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
48                                 GlobalValue::PrivateLinkage, StrConstant, Name,
49                                 nullptr, GlobalVariable::NotThreadLocal,
50                                 AddressSpace);
51   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
52   GV->setAlignment(Align(1));
53   return GV;
54 }
55 
56 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
57   assert(BB && BB->getParent() && "No current function!");
58   return BB->getParent()->getReturnType();
59 }
60 
61 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
62   auto *PT = cast<PointerType>(Ptr->getType());
63   if (PT->getElementType()->isIntegerTy(8))
64     return Ptr;
65 
66   // Otherwise, we need to insert a bitcast.
67   PT = getInt8PtrTy(PT->getAddressSpace());
68   BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
69   BB->getInstList().insert(InsertPt, BCI);
70   SetInstDebugLocation(BCI);
71   return BCI;
72 }
73 
74 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
75                                   IRBuilderBase *Builder,
76                                   const Twine &Name = "",
77                                   Instruction *FMFSource = nullptr) {
78   CallInst *CI = CallInst::Create(Callee, Ops, Name);
79   if (FMFSource)
80     CI->copyFastMathFlags(FMFSource);
81   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
82   Builder->SetInstDebugLocation(CI);
83   return CI;
84 }
85 
86 static InvokeInst *createInvokeHelper(Function *Invokee, BasicBlock *NormalDest,
87                                       BasicBlock *UnwindDest,
88                                       ArrayRef<Value *> Ops,
89                                       IRBuilderBase *Builder,
90                                       const Twine &Name = "") {
91   InvokeInst *II =
92       InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
93   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
94                                                   II);
95   Builder->SetInstDebugLocation(II);
96   return II;
97 }
98 
99 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
100                                       MaybeAlign Align, bool isVolatile,
101                                       MDNode *TBAATag, MDNode *ScopeTag,
102                                       MDNode *NoAliasTag) {
103   Ptr = getCastedInt8PtrValue(Ptr);
104   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
105   Type *Tys[] = { Ptr->getType(), Size->getType() };
106   Module *M = BB->getParent()->getParent();
107   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
108 
109   CallInst *CI = createCallHelper(TheFn, Ops, this);
110 
111   if (Align)
112     cast<MemSetInst>(CI)->setDestAlignment(Align->value());
113 
114   // Set the TBAA info if present.
115   if (TBAATag)
116     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
117 
118   if (ScopeTag)
119     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
120 
121   if (NoAliasTag)
122     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
123 
124   return CI;
125 }
126 
127 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
128     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
129     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
130 
131   Ptr = getCastedInt8PtrValue(Ptr);
132   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
133   Type *Tys[] = {Ptr->getType(), Size->getType()};
134   Module *M = BB->getParent()->getParent();
135   Function *TheFn = Intrinsic::getDeclaration(
136       M, Intrinsic::memset_element_unordered_atomic, Tys);
137 
138   CallInst *CI = createCallHelper(TheFn, Ops, this);
139 
140   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
141 
142   // Set the TBAA info if present.
143   if (TBAATag)
144     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
145 
146   if (ScopeTag)
147     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
148 
149   if (NoAliasTag)
150     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
151 
152   return CI;
153 }
154 
155 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,
156                                       unsigned SrcAlign, Value *Size,
157                                       bool isVolatile, MDNode *TBAATag,
158                                       MDNode *TBAAStructTag, MDNode *ScopeTag,
159                                       MDNode *NoAliasTag) {
160   return CreateMemCpy(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign),
161                       Size, isVolatile, TBAATag, TBAAStructTag, ScopeTag,
162                       NoAliasTag);
163 }
164 
165 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign,
166                                       Value *Src, MaybeAlign SrcAlign,
167                                       Value *Size, bool isVolatile,
168                                       MDNode *TBAATag, MDNode *TBAAStructTag,
169                                       MDNode *ScopeTag, MDNode *NoAliasTag) {
170   Dst = getCastedInt8PtrValue(Dst);
171   Src = getCastedInt8PtrValue(Src);
172 
173   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
174   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
175   Module *M = BB->getParent()->getParent();
176   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
177 
178   CallInst *CI = createCallHelper(TheFn, Ops, this);
179 
180   auto* MCI = cast<MemCpyInst>(CI);
181   if (DstAlign)
182     MCI->setDestAlignment(*DstAlign);
183   if (SrcAlign)
184     MCI->setSourceAlignment(*SrcAlign);
185 
186   // Set the TBAA info if present.
187   if (TBAATag)
188     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
189 
190   // Set the TBAA Struct info if present.
191   if (TBAAStructTag)
192     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
193 
194   if (ScopeTag)
195     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
196 
197   if (NoAliasTag)
198     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
199 
200   return CI;
201 }
202 
203 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
204     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
205     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
206     MDNode *ScopeTag, MDNode *NoAliasTag) {
207   assert(DstAlign >= ElementSize &&
208          "Pointer alignment must be at least element size");
209   assert(SrcAlign >= ElementSize &&
210          "Pointer alignment must be at least element size");
211   Dst = getCastedInt8PtrValue(Dst);
212   Src = getCastedInt8PtrValue(Src);
213 
214   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
215   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
216   Module *M = BB->getParent()->getParent();
217   Function *TheFn = Intrinsic::getDeclaration(
218       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
219 
220   CallInst *CI = createCallHelper(TheFn, Ops, this);
221 
222   // Set the alignment of the pointer args.
223   auto *AMCI = cast<AtomicMemCpyInst>(CI);
224   AMCI->setDestAlignment(DstAlign);
225   AMCI->setSourceAlignment(SrcAlign);
226 
227   // Set the TBAA info if present.
228   if (TBAATag)
229     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
230 
231   // Set the TBAA Struct info if present.
232   if (TBAAStructTag)
233     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
234 
235   if (ScopeTag)
236     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
237 
238   if (NoAliasTag)
239     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
240 
241   return CI;
242 }
243 
244 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
245                                        Value *Src, MaybeAlign SrcAlign,
246                                        Value *Size, bool isVolatile,
247                                        MDNode *TBAATag, MDNode *ScopeTag,
248                                        MDNode *NoAliasTag) {
249   Dst = getCastedInt8PtrValue(Dst);
250   Src = getCastedInt8PtrValue(Src);
251 
252   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
253   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
254   Module *M = BB->getParent()->getParent();
255   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
256 
257   CallInst *CI = createCallHelper(TheFn, Ops, this);
258 
259   auto *MMI = cast<MemMoveInst>(CI);
260   if (DstAlign)
261     MMI->setDestAlignment(*DstAlign);
262   if (SrcAlign)
263     MMI->setSourceAlignment(*SrcAlign);
264 
265   // Set the TBAA info if present.
266   if (TBAATag)
267     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
268 
269   if (ScopeTag)
270     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
271 
272   if (NoAliasTag)
273     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
274 
275   return CI;
276 }
277 
278 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
279     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
280     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
281     MDNode *ScopeTag, MDNode *NoAliasTag) {
282   assert(DstAlign >= ElementSize &&
283          "Pointer alignment must be at least element size");
284   assert(SrcAlign >= ElementSize &&
285          "Pointer alignment must be at least element size");
286   Dst = getCastedInt8PtrValue(Dst);
287   Src = getCastedInt8PtrValue(Src);
288 
289   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
290   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
291   Module *M = BB->getParent()->getParent();
292   Function *TheFn = Intrinsic::getDeclaration(
293       M, Intrinsic::memmove_element_unordered_atomic, Tys);
294 
295   CallInst *CI = createCallHelper(TheFn, Ops, this);
296 
297   // Set the alignment of the pointer args.
298   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
299   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
300 
301   // Set the TBAA info if present.
302   if (TBAATag)
303     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
304 
305   // Set the TBAA Struct info if present.
306   if (TBAAStructTag)
307     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
308 
309   if (ScopeTag)
310     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
311 
312   if (NoAliasTag)
313     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
314 
315   return CI;
316 }
317 
318 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
319                                     Value *Src) {
320   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
321   Value *Ops[] = {Src};
322   Type *Tys[] = { Src->getType() };
323   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
324   return createCallHelper(Decl, Ops, Builder);
325 }
326 
327 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
328   Module *M = GetInsertBlock()->getParent()->getParent();
329   Value *Ops[] = {Acc, Src};
330   Type *Tys[] = {Acc->getType(), Src->getType()};
331   auto Decl = Intrinsic::getDeclaration(
332       M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
333   return createCallHelper(Decl, Ops, this);
334 }
335 
336 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
337   Module *M = GetInsertBlock()->getParent()->getParent();
338   Value *Ops[] = {Acc, Src};
339   Type *Tys[] = {Acc->getType(), Src->getType()};
340   auto Decl = Intrinsic::getDeclaration(
341       M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
342   return createCallHelper(Decl, Ops, this);
343 }
344 
345 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
346   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
347                                Src);
348 }
349 
350 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
351   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
352                                Src);
353 }
354 
355 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
356   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
357                                Src);
358 }
359 
360 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
361   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
362                                Src);
363 }
364 
365 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
366   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
367                                Src);
368 }
369 
370 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
371   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
372                      : Intrinsic::experimental_vector_reduce_umax;
373   return getReductionIntrinsic(this, ID, Src);
374 }
375 
376 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
377   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
378                      : Intrinsic::experimental_vector_reduce_umin;
379   return getReductionIntrinsic(this, ID, Src);
380 }
381 
382 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
383   auto Rdx = getReductionIntrinsic(
384       this, Intrinsic::experimental_vector_reduce_fmax, Src);
385   if (NoNaN) {
386     FastMathFlags FMF;
387     FMF.setNoNaNs();
388     Rdx->setFastMathFlags(FMF);
389   }
390   return Rdx;
391 }
392 
393 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
394   auto Rdx = getReductionIntrinsic(
395       this, Intrinsic::experimental_vector_reduce_fmin, Src);
396   if (NoNaN) {
397     FastMathFlags FMF;
398     FMF.setNoNaNs();
399     Rdx->setFastMathFlags(FMF);
400   }
401   return Rdx;
402 }
403 
404 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
405   assert(isa<PointerType>(Ptr->getType()) &&
406          "lifetime.start only applies to pointers.");
407   Ptr = getCastedInt8PtrValue(Ptr);
408   if (!Size)
409     Size = getInt64(-1);
410   else
411     assert(Size->getType() == getInt64Ty() &&
412            "lifetime.start requires the size to be an i64");
413   Value *Ops[] = { Size, Ptr };
414   Module *M = BB->getParent()->getParent();
415   Function *TheFn =
416       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
417   return createCallHelper(TheFn, Ops, this);
418 }
419 
420 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
421   assert(isa<PointerType>(Ptr->getType()) &&
422          "lifetime.end only applies to pointers.");
423   Ptr = getCastedInt8PtrValue(Ptr);
424   if (!Size)
425     Size = getInt64(-1);
426   else
427     assert(Size->getType() == getInt64Ty() &&
428            "lifetime.end requires the size to be an i64");
429   Value *Ops[] = { Size, Ptr };
430   Module *M = BB->getParent()->getParent();
431   Function *TheFn =
432       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
433   return createCallHelper(TheFn, Ops, this);
434 }
435 
436 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
437 
438   assert(isa<PointerType>(Ptr->getType()) &&
439          "invariant.start only applies to pointers.");
440   Ptr = getCastedInt8PtrValue(Ptr);
441   if (!Size)
442     Size = getInt64(-1);
443   else
444     assert(Size->getType() == getInt64Ty() &&
445            "invariant.start requires the size to be an i64");
446 
447   Value *Ops[] = {Size, Ptr};
448   // Fill in the single overloaded type: memory object type.
449   Type *ObjectPtr[1] = {Ptr->getType()};
450   Module *M = BB->getParent()->getParent();
451   Function *TheFn =
452       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
453   return createCallHelper(TheFn, Ops, this);
454 }
455 
456 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
457   assert(Cond->getType() == getInt1Ty() &&
458          "an assumption condition must be of type i1");
459 
460   Value *Ops[] = { Cond };
461   Module *M = BB->getParent()->getParent();
462   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
463   return createCallHelper(FnAssume, Ops, this);
464 }
465 
466 /// Create a call to a Masked Load intrinsic.
467 /// \p Ptr       - base pointer for the load
468 /// \p Alignment - alignment of the source location
469 /// \p Mask      - vector of booleans which indicates what vector lanes should
470 ///                be accessed in memory
471 /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
472 ///                of the result
473 /// \p Name      - name of the result variable
474 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment,
475                                           Value *Mask, Value *PassThru,
476                                           const Twine &Name) {
477   auto *PtrTy = cast<PointerType>(Ptr->getType());
478   Type *DataTy = PtrTy->getElementType();
479   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
480   assert(Mask && "Mask should not be all-ones (null)");
481   if (!PassThru)
482     PassThru = UndefValue::get(DataTy);
483   Type *OverloadedTypes[] = { DataTy, PtrTy };
484   Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
485   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
486                                OverloadedTypes, Name);
487 }
488 
489 /// Create a call to a Masked Store intrinsic.
490 /// \p Val       - data to be stored,
491 /// \p Ptr       - base pointer for the store
492 /// \p Alignment - alignment of the destination location
493 /// \p Mask      - vector of booleans which indicates what vector lanes should
494 ///                be accessed in memory
495 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
496                                            Align Alignment, Value *Mask) {
497   auto *PtrTy = cast<PointerType>(Ptr->getType());
498   Type *DataTy = PtrTy->getElementType();
499   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
500   assert(Mask && "Mask should not be all-ones (null)");
501   Type *OverloadedTypes[] = { DataTy, PtrTy };
502   Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
503   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
504 }
505 
506 /// Create a call to a Masked intrinsic, with given intrinsic Id,
507 /// an array of operands - Ops, and an array of overloaded types -
508 /// OverloadedTypes.
509 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
510                                                ArrayRef<Value *> Ops,
511                                                ArrayRef<Type *> OverloadedTypes,
512                                                const Twine &Name) {
513   Module *M = BB->getParent()->getParent();
514   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
515   return createCallHelper(TheFn, Ops, this, Name);
516 }
517 
518 /// Create a call to a Masked Gather intrinsic.
519 /// \p Ptrs     - vector of pointers for loading
520 /// \p Align    - alignment for one element
521 /// \p Mask     - vector of booleans which indicates what vector lanes should
522 ///               be accessed in memory
523 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
524 ///               of the result
525 /// \p Name     - name of the result variable
526 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
527                                             Value *Mask, Value *PassThru,
528                                             const Twine &Name) {
529   auto PtrsTy = cast<VectorType>(Ptrs->getType());
530   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
531   unsigned NumElts = PtrsTy->getVectorNumElements();
532   Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
533 
534   if (!Mask)
535     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
536                                      NumElts));
537 
538   if (!PassThru)
539     PassThru = UndefValue::get(DataTy);
540 
541   Type *OverloadedTypes[] = {DataTy, PtrsTy};
542   Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
543 
544   // We specify only one type when we create this intrinsic. Types of other
545   // arguments are derived from this type.
546   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
547                                Name);
548 }
549 
550 /// Create a call to a Masked Scatter intrinsic.
551 /// \p Data  - data to be stored,
552 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
553 ///            stored
554 /// \p Align - alignment for one element
555 /// \p Mask  - vector of booleans which indicates what vector lanes should
556 ///            be accessed in memory
557 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
558                                              Align Alignment, Value *Mask) {
559   auto PtrsTy = cast<VectorType>(Ptrs->getType());
560   auto DataTy = cast<VectorType>(Data->getType());
561   unsigned NumElts = PtrsTy->getVectorNumElements();
562 
563 #ifndef NDEBUG
564   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
565   assert(NumElts == DataTy->getVectorNumElements() &&
566          PtrTy->getElementType() == DataTy->getElementType() &&
567          "Incompatible pointer and data types");
568 #endif
569 
570   if (!Mask)
571     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
572                                      NumElts));
573 
574   Type *OverloadedTypes[] = {DataTy, PtrsTy};
575   Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
576 
577   // We specify only one type when we create this intrinsic. Types of other
578   // arguments are derived from this type.
579   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
580 }
581 
582 template <typename T0, typename T1, typename T2, typename T3>
583 static std::vector<Value *>
584 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
585                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
586                   ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
587                   ArrayRef<T3> GCArgs) {
588   std::vector<Value *> Args;
589   Args.push_back(B.getInt64(ID));
590   Args.push_back(B.getInt32(NumPatchBytes));
591   Args.push_back(ActualCallee);
592   Args.push_back(B.getInt32(CallArgs.size()));
593   Args.push_back(B.getInt32(Flags));
594   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
595   Args.push_back(B.getInt32(TransitionArgs.size()));
596   Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
597   Args.push_back(B.getInt32(DeoptArgs.size()));
598   Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
599   Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
600 
601   return Args;
602 }
603 
604 template <typename T0, typename T1, typename T2, typename T3>
605 static CallInst *CreateGCStatepointCallCommon(
606     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
607     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
608     ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
609     const Twine &Name) {
610   // Extract out the type of the callee.
611   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
612   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
613          "actual callee must be a callable value");
614 
615   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
616   // Fill in the one generic type'd argument (the function is also vararg)
617   Type *ArgTypes[] = { FuncPtrType };
618   Function *FnStatepoint =
619     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
620                               ArgTypes);
621 
622   std::vector<Value *> Args =
623       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
624                         CallArgs, TransitionArgs, DeoptArgs, GCArgs);
625   return createCallHelper(FnStatepoint, Args, Builder, Name);
626 }
627 
628 CallInst *IRBuilderBase::CreateGCStatepointCall(
629     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
630     ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
631     ArrayRef<Value *> GCArgs, const Twine &Name) {
632   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
633       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
634       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
635 }
636 
637 CallInst *IRBuilderBase::CreateGCStatepointCall(
638     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
639     ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
640     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
641   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
642       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
643       DeoptArgs, GCArgs, Name);
644 }
645 
646 CallInst *IRBuilderBase::CreateGCStatepointCall(
647     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
648     ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
649     ArrayRef<Value *> GCArgs, const Twine &Name) {
650   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
651       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
652       CallArgs, None, DeoptArgs, GCArgs, Name);
653 }
654 
655 template <typename T0, typename T1, typename T2, typename T3>
656 static InvokeInst *CreateGCStatepointInvokeCommon(
657     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
658     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
659     uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
660     ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
661   // Extract out the type of the callee.
662   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
663   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
664          "actual callee must be a callable value");
665 
666   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
667   // Fill in the one generic type'd argument (the function is also vararg)
668   Function *FnStatepoint = Intrinsic::getDeclaration(
669       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
670 
671   std::vector<Value *> Args =
672       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
673                         InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
674   return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
675                             Name);
676 }
677 
678 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
679     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
680     BasicBlock *NormalDest, BasicBlock *UnwindDest,
681     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
682     ArrayRef<Value *> GCArgs, const Twine &Name) {
683   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
684       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
685       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
686       DeoptArgs, GCArgs, Name);
687 }
688 
689 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
690     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
691     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
692     ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
693     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
694   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
695       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
696       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
697 }
698 
699 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
700     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
701     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
702     ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
703   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
704       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
705       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
706       Name);
707 }
708 
709 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
710                                        Type *ResultType,
711                                        const Twine &Name) {
712  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
713  Module *M = BB->getParent()->getParent();
714  Type *Types[] = {ResultType};
715  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
716 
717  Value *Args[] = {Statepoint};
718  return createCallHelper(FnGCResult, Args, this, Name);
719 }
720 
721 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
722                                          int BaseOffset,
723                                          int DerivedOffset,
724                                          Type *ResultType,
725                                          const Twine &Name) {
726  Module *M = BB->getParent()->getParent();
727  Type *Types[] = {ResultType};
728  Function *FnGCRelocate =
729      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
730 
731  Value *Args[] = {Statepoint,
732                   getInt32(BaseOffset),
733                   getInt32(DerivedOffset)};
734  return createCallHelper(FnGCRelocate, Args, this, Name);
735 }
736 
737 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
738                                               Instruction *FMFSource,
739                                               const Twine &Name) {
740   Module *M = BB->getModule();
741   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
742   return createCallHelper(Fn, {V}, this, Name, FMFSource);
743 }
744 
745 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
746                                                Value *RHS,
747                                                Instruction *FMFSource,
748                                                const Twine &Name) {
749   Module *M = BB->getModule();
750   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
751   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
752 }
753 
754 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
755                                          ArrayRef<Type *> Types,
756                                          ArrayRef<Value *> Args,
757                                          Instruction *FMFSource,
758                                          const Twine &Name) {
759   Module *M = BB->getModule();
760   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
761   return createCallHelper(Fn, Args, this, Name, FMFSource);
762 }
763